/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. */ #ifndef _SYS_IB_CLIENTS_IBD_H #define _SYS_IB_CLIENTS_IBD_H #ifdef __cplusplus extern "C" { #endif /* The following macros are used in both ibd.c and ibd_cm.c */ /* * Completion queue polling control */ #define IBD_CQ_POLLING 0x1 #define IBD_REDO_CQ_POLLING 0x2 /* * Maximum length for returning chained mps back to crossbow. * Also used as the maximum number of rx wc's polled at a time. */ #define IBD_MAX_RX_MP_LEN 16 /* * When doing multiple-send-wr, this value determines how many to do at * a time (in a single ibt_post_send). */ #define IBD_MAX_TX_POST_MULTIPLE 4 /* * Flag bits for resources to reap */ #define IBD_RSRC_SWQE 0x1 #define IBD_RSRC_LSOBUF 0x2 #define IBD_RSRC_RC_SWQE 0x4 #define IBD_RSRC_RC_TX_LARGEBUF 0x8 /* * Async operation types */ #define IBD_ASYNC_GETAH 1 #define IBD_ASYNC_JOIN 2 #define IBD_ASYNC_LEAVE 3 #define IBD_ASYNC_PROMON 4 #define IBD_ASYNC_PROMOFF 5 #define IBD_ASYNC_REAP 6 #define IBD_ASYNC_TRAP 7 #define IBD_ASYNC_SCHED 8 #define IBD_ASYNC_LINK 9 #define IBD_ASYNC_EXIT 10 #define IBD_ASYNC_RC_TOO_BIG 11 #define IBD_ASYNC_RC_CLOSE_ACT_CHAN 12 #define IBD_ASYNC_RC_RECYCLE_ACE 13 #define IBD_ASYNC_RC_CLOSE_PAS_CHAN 14 /* * State of IBD driver initialization during attach/m_start */ #define IBD_DRV_STATE_INITIALIZED 0x000001 #define IBD_DRV_RXINTR_ADDED 0x000002 #define IBD_DRV_TXINTR_ADDED 0x000004 #define IBD_DRV_IBTL_ATTACH_DONE 0x000008 #define IBD_DRV_HCA_OPENED 0x000010 #define IBD_DRV_PD_ALLOCD 0x000020 #define IBD_DRV_MAC_REGISTERED 0x000040 #define IBD_DRV_PORT_DETAILS_OBTAINED 0x000080 #define IBD_DRV_BCAST_GROUP_FOUND 0x000100 #define IBD_DRV_ACACHE_INITIALIZED 0x000200 #define IBD_DRV_CQS_ALLOCD 0x000400 #define IBD_DRV_UD_CHANNEL_SETUP 0x000800 #define IBD_DRV_TXLIST_ALLOCD 0x001000 #define IBD_DRV_SCQ_NOTIFY_ENABLED 0x002000 #define IBD_DRV_RXLIST_ALLOCD 0x004000 #define IBD_DRV_BCAST_GROUP_JOINED 0x008000 #define IBD_DRV_ASYNC_THR_CREATED 0x010000 #define IBD_DRV_RCQ_NOTIFY_ENABLED 0x020000 #define IBD_DRV_SM_NOTICES_REGISTERED 0x040000 #define IBD_DRV_STARTED 0x080000 #define IBD_DRV_RC_SRQ_ALLOCD 0x100000 #define IBD_DRV_RC_LARGEBUF_ALLOCD 0x200000 #define IBD_DRV_RC_LISTEN 0x400000 #ifdef DEBUG #define IBD_DRV_RC_PRIVATE_STATE 0x800000 #endif #define IBD_DRV_IN_DELETION 0x1000000 #define IBD_DRV_IN_LATE_HCA_INIT 0x2000000 #define IBD_DRV_REQ_LIST_INITED 0x4000000 #define IBD_DRV_RC_TIMEOUT 0x8000000 /* * Miscellaneous constants */ #define IBD_SEND 0 #define IBD_RECV 1 /* Tunables defaults and limits */ #define IBD_LINK_MODE_UD 0 #define IBD_LINK_MODE_RC 1 #define IBD_DEF_LINK_MODE IBD_LINK_MODE_RC #define IBD_DEF_LSO_POLICY B_TRUE #define IBD_DEF_NUM_LSO_BUFS 1024 #define IBD_DEF_CREATE_BCAST_GROUP B_TRUE #define IBD_DEF_COALESCE_COMPLETIONS B_TRUE #define IBD_DEF_UD_RX_COMP_COUNT 4 #define IBD_DEF_UD_RX_COMP_USEC 10 #define IBD_DEF_UD_TX_COMP_COUNT 16 #define IBD_DEF_UD_TX_COMP_USEC 300 #define IBD_DEF_RC_RX_COMP_COUNT 4 #define IBD_DEF_RC_RX_COMP_USEC 10 #define IBD_DEF_RC_TX_COMP_COUNT 10 #define IBD_DEF_RC_TX_COMP_USEC 300 #define IBD_DEF_UD_TX_COPY_THRESH 4096 #define IBD_DEF_RC_RX_COPY_THRESH 4096 #define IBD_DEF_RC_TX_COPY_THRESH 4096 #define IBD_DEF_UD_NUM_RWQE 4000 #define IBD_DEF_UD_NUM_SWQE 4000 #define IBD_DEF_RC_ENABLE_SRQ B_TRUE #if defined(__i386) #define IBD_DEF_RC_NUM_RWQE 511 #define IBD_DEF_RC_NUM_SWQE 255 #else #define IBD_DEF_RC_NUM_RWQE 2047 #define IBD_DEF_RC_NUM_SWQE 511 #endif #define IBD_DEF_NUM_AH 256 #define IBD_DEF_HASH_SIZE 32 #define IBD_DEF_RC_NUM_SRQ (IBD_DEF_RC_NUM_RWQE - 1) #define IBD_DEF_RC_RX_RWQE_THRESH (IBD_DEF_RC_NUM_RWQE >> 2) /* Tunable limits */ #define IBD_MIN_NUM_LSO_BUFS 512 #define IBD_MAX_NUM_LSO_BUFS 4096 #define IBD_MIN_UD_TX_COPY_THRESH 2048 #define IBD_MAX_UD_TX_COPY_THRESH 65536 #define IBD_MIN_UD_NUM_SWQE 512 #define IBD_MAX_UD_NUM_SWQE 8000 #define IBD_MIN_UD_NUM_RWQE 512 #define IBD_MAX_UD_NUM_RWQE 8000 #define IBD_MIN_NUM_AH 32 #define IBD_MAX_NUM_AH 8192 #define IBD_MIN_HASH_SIZE 32 #define IBD_MAX_HASH_SIZE 1024 #if defined(__i386) #define IBD_MIN_RC_NUM_SWQE 255 #else #define IBD_MIN_RC_NUM_SWQE 511 #endif #define IBD_MAX_RC_NUM_SWQE 8000 #define IBD_MIN_RC_NUM_RWQE 511 #define IBD_MAX_RC_NUM_RWQE 8000 #define IBD_MIN_RC_RX_COPY_THRESH 1500 #define IBD_MAX_RC_RX_COPY_THRESH 65520 #define IBD_MIN_RC_TX_COPY_THRESH 1500 #define IBD_MAX_RC_TX_COPY_THRESH 65520 #define IBD_MIN_RC_NUM_SRQ (IBD_MIN_RC_NUM_RWQE - 1) #define IBD_MIN_RC_RX_RWQE_THRESH (IBD_MIN_RC_NUM_RWQE >> 2) /* * Thresholds * * When waiting for resources (swqes or lso buffers) to become available, * the first two thresholds below determine how long to wait before informing * the network layer to start sending packets again. The IBD_TX_POLL_THRESH * determines how low the available swqes should go before we start polling * the completion queue. */ #define IBD_FREE_LSOS_THRESH 8 #define IBD_FREE_SWQES_THRESH 20 #define IBD_TX_POLL_THRESH 80 #ifdef DEBUG void debug_print(int l, char *fmt, ...); #define DPRINT debug_print #else #define DPRINT 0 && #endif /* * AH and MCE active list manipulation: * * Multicast disable requests and MCG delete traps are two cases * where the active AH entry for the mcg (if any unreferenced one exists) * will be moved to the free list (to force the next Tx to the mcg to * join the MCG in SendOnly mode). Port up handling will also move AHs * from active to free list. * * In the case when some transmits are still pending on an entry * for an mcg, but a multicast disable has already been issued on the * mcg, there are some options to consider to preserve the join state * to ensure the emitted packet is properly routed on the IBA fabric. * For the AH, we can * 1. take out of active list at multicast disable time. * 2. take out of active list only when last pending Tx completes. * For the MCE, we can * 3. take out of active list at multicast disable time. * 4. take out of active list only when last pending Tx completes. * 5. move from active list to stale list at multicast disable time. * We choose to use 2,4. We use option 4 so that if a multicast enable * is tried before the pending Tx completes, the enable code finds the * mce in the active list and just has to make sure it will not be reaped * (ie the mcg leave done) when the pending Tx does complete. Alternatively, * a stale list (#5) that would be checked in the enable code would need * to be implemented. Option 2 is used, because otherwise, a Tx attempt * after the multicast disable would try to put an AH in the active list, * and associate the mce it finds in the active list to this new AH, * whereas the mce is already associated with the previous AH (taken off * the active list), and will be removed once the pending Tx's complete * (unless a reference count on mce's is implemented). One implication of * using 2,4 is that new Tx's posted before the pending Tx's complete will * grab new references on the AH, further delaying the leave. * * In the case of mcg delete (or create) trap when the port is sendonly * joined, the AH and MCE handling is different: the AH and MCE has to be * immediately taken off the active lists (forcing a join and path lookup * at the next Tx is the only guaranteed means of ensuring a proper Tx * to an mcg as it is repeatedly created and deleted and goes thru * reincarnations). * * When a port is already sendonly joined, and a multicast enable is * attempted, the same mce structure is promoted; this ensures only a * single mce on the active list tracks the most powerful join state. * * In the case of port up event handling, the MCE for sendonly membership * is freed up, and the ACE is put into the free list as soon as possible * (depending on whether posted Tx's have completed). For fullmembership * MCE's though, the ACE is similarly handled; but the MCE is kept around * (a re-JOIN is attempted) only if the DLPI leave has not already been * done; else the mce is deconstructed (mc_fullreap case). * * MCG creation and deletion trap handling: * * These traps are unreliable (meaning sometimes the trap might never * be delivered to the subscribed nodes) and may arrive out-of-order * since they use UD transport. An alternative to relying on these * unreliable traps is to poll for mcg presence every so often, but * instead of doing that, we try to be as conservative as possible * while handling the traps, and hope that the traps do arrive at * the subscribed nodes soon. Note that if a node is fullmember * joined to an mcg, it can not possibly receive a mcg create/delete * trap for that mcg (by fullmember definition); if it does, it is * an old trap from a previous incarnation of the mcg. * * Whenever a trap is received, the driver cleans up its sendonly * membership to the group; we choose to do a sendonly leave even * on a creation trap to handle the case of a prior deletion of the mcg * having gone unnoticed. Consider an example scenario: * T1: MCG M is deleted, and fires off deletion trap D1. * T2: MCG M is recreated, fires off creation trap C1, which is lost. * T3: Node N tries to transmit to M, joining in sendonly mode. * T4: MCG M is deleted, and fires off deletion trap D2. * T5: N receives a deletion trap, but can not distinguish D1 from D2. * If the trap is D2, then a LEAVE is not required, since the mcg * is already deleted; but if it is D1, a LEAVE is required. A safe * approach is to always LEAVE, but the SM may be confused if it * receives a LEAVE without a prior JOIN. * * Management of the non-membership to an mcg is similar to the above, * except that if the interface is in promiscuous mode, it is required * to attempt to re-join the mcg after receiving a trap. Unfortunately, * if the re-join attempt fails (in which case a warning message needs * to be printed), it is not clear whether it failed due to the mcg not * existing, or some fabric/hca issues, due to the delayed nature of * trap delivery. Querying the SA to establish presence/absence of the * mcg is also racy at best. Thus, the driver just prints a warning * message when it can not rejoin after receiving a create trap, although * this might be (on rare occasions) a mis-warning if the create trap is * received after the mcg was deleted. */ /* * Implementation of atomic "recycle" bits and reference count * on address handles. This utilizes the fact that max reference * count on any handle is limited by number of send wqes, thus * high bits in the ac_ref field can be used as the recycle bits, * and only the low bits hold the number of pending Tx requests. * This atomic AH reference counting allows the Tx completion * handler not to acquire the id_ac_mutex to process every completion, * thus reducing lock contention problems between completion and * the Tx path. */ #define CYCLEVAL 0x80000 #define CLEAR_REFCYCLE(ace) (ace)->ac_ref = 0 #define CYCLE_SET(ace) (((ace)->ac_ref & CYCLEVAL) == CYCLEVAL) #define GET_REF(ace) ((ace)->ac_ref) #define GET_REF_CYCLE(ace) ( \ /* \ * Make sure "cycle" bit is set. \ */ \ ASSERT(CYCLE_SET(ace)), \ ((ace)->ac_ref & ~(CYCLEVAL)) \ ) #define INC_REF(ace, num) { \ atomic_add_32(&(ace)->ac_ref, num); \ } #define SET_CYCLE_IF_REF(ace) ( \ CYCLE_SET(ace) ? B_TRUE : \ atomic_add_32_nv(&ace->ac_ref, CYCLEVAL) == \ CYCLEVAL ? \ /* \ * Clear the "cycle" bit we just set; \ * ref count known to be 0 from above. \ */ \ CLEAR_REFCYCLE(ace), B_FALSE : \ /* \ * We set "cycle" bit; let caller know. \ */ \ B_TRUE \ ) #define DEC_REF_DO_CYCLE(ace) ( \ atomic_dec_32_nv(&ace->ac_ref) == CYCLEVAL ? \ /* \ * Ref count known to be 0 from above. \ */ \ B_TRUE : \ B_FALSE \ ) /* * Address handle entries maintained by the driver are kept in the * free and active lists. Each entry starts out in the free list; * it migrates to the active list when primed using ibt_get_paths() * and ibt_modify_ud_dest() for transmission to a specific destination. * In the active list, the entry has a reference count indicating the * number of ongoing/uncompleted transmits that reference it. The * entry is left in the active list even after the reference count * goes to 0, since successive transmits can find it there and do * not need to set up another entry (ie the path information is * cached using the active list). Entries on the active list are * also hashed using the destination link address as a key for faster * lookups during transmits. * * For any destination address (unicast or multicast, whatever the * join states), there will be at most one entry in the active list. * Entries with a 0 reference count on the active list can be reused * for a transmit to a new destination, if the free list is empty. * * The AH free list insertion/deletion is protected with the id_ac_mutex, * since the async thread and Tx callback handlers insert/delete. The * active list does not need a lock (all operations are done by the * async thread) but updates to the reference count are atomically * done (increments done by Tx path, decrements by the Tx callback handler). */ #define IBD_ACACHE_INSERT_FREE(state, ce) \ list_insert_head(&state->id_ah_free, ce) #define IBD_ACACHE_GET_FREE(state) \ list_get_head(&state->id_ah_free) #define IBD_ACACHE_INSERT_ACTIVE(state, ce) { \ int _ret_; \ list_insert_head(&state->id_ah_active, ce); \ _ret_ = mod_hash_insert(state->id_ah_active_hash, \ (mod_hash_key_t)&ce->ac_mac, (mod_hash_val_t)ce); \ ASSERT(_ret_ == 0); \ state->id_ac_hot_ace = ce; \ } #define IBD_ACACHE_PULLOUT_ACTIVE(state, ce) { \ list_remove(&state->id_ah_active, ce); \ if (state->id_ac_hot_ace == ce) \ state->id_ac_hot_ace = NULL; \ (void) mod_hash_remove(state->id_ah_active_hash, \ (mod_hash_key_t)&ce->ac_mac, (mod_hash_val_t)ce); \ } #define IBD_ACACHE_GET_ACTIVE(state) \ list_get_head(&state->id_ah_active) /* * Padding for nd6 Neighbor Solicitation and Advertisement needs to be at * front of optional src/tgt link layer address. Right now Solaris inserts * padding by default at the end. The routine which is doing is nce_xmit() * in ip_ndp.c. It copies the nd_lla_addr after the nd_opt_hdr_t. So when * the packet comes down from IP layer to the IBD driver, it is in the * following format: [IPoIB_PTXHDR_T][INET6 packet][ICMP6][OPT_ND_HDR_T] * This size is 2 bytes followed by [22 bytes of ipoib_machdr]. As a result * machdr is not 4 byte aligned and had 2 bytes of padding at the end. * * The send routine at IBD driver changes this packet as follows: * [IPoIB_HDR_T][INET6 packet][ICMP6][OPT_ND_HDR_T + 2 bytes of padding] * followed by [22 bytes of ipoib_machdr] resulting in machdr 4 byte * aligned. * * At the receiving side again ibd_process_rx takes the above packet and * removes the two bytes of front padding and inserts it at the end. This * is since the IP layer does not understand padding at the front. */ #define IBD_PAD_NSNA(ip6h, len, type) { \ uchar_t *nd_lla_ptr; \ icmp6_t *icmp6; \ nd_opt_hdr_t *opt; \ int i; \ \ icmp6 = (icmp6_t *)&ip6h[1]; \ len -= sizeof (nd_neighbor_advert_t); \ if (((icmp6->icmp6_type == ND_NEIGHBOR_SOLICIT) || \ (icmp6->icmp6_type == ND_NEIGHBOR_ADVERT)) && \ (len != 0)) { \ opt = (nd_opt_hdr_t *)((uint8_t *)ip6h \ + IPV6_HDR_LEN + sizeof (nd_neighbor_advert_t)); \ ASSERT(opt != NULL); \ nd_lla_ptr = (uchar_t *)&opt[1]; \ if (type == IBD_SEND) { \ for (i = IPOIB_ADDRL; i > 0; i--) \ *(nd_lla_ptr + i + 1) = \ *(nd_lla_ptr + i - 1); \ } else { \ for (i = 0; i < IPOIB_ADDRL; i++) \ *(nd_lla_ptr + i) = \ *(nd_lla_ptr + i + 2); \ } \ *(nd_lla_ptr + i) = 0; \ *(nd_lla_ptr + i + 1) = 0; \ } \ } /* * IETF defined IPoIB encapsulation header, with 2b of ethertype * followed by 2 reserved bytes. This is at the start of the * datagram sent to and received over the wire by the driver. */ typedef struct ipoib_header { ushort_t ipoib_type; ushort_t ipoib_mbz; } ipoib_hdr_t; #define IPOIB_HDRSIZE sizeof (struct ipoib_header) /* * IETF defined IPoIB link address; IBA QPN, followed by GID, * which has a prefix and suffix, as reported via ARP. */ typedef struct ipoib_mac { uint32_t ipoib_qpn; uint32_t ipoib_gidpref[2]; uint32_t ipoib_gidsuff[2]; } ipoib_mac_t; #define IPOIB_ADDRL sizeof (struct ipoib_mac) /* * Pseudo header prepended to datagram in DLIOCRAW transmit path * and when GLD hands the datagram to the gldm_send entry point. */ typedef struct ipoib_ptxhdr { ipoib_mac_t ipoib_dest; ipoib_hdr_t ipoib_rhdr; } ipoib_ptxhdr_t; #define IPOIBDLSAP(p, offset) ((ipoib_ptxhdr_t *)((caddr_t)(p)+offset)) /* * The pseudo-GRH structure that sits before the data in the * receive buffer, and is overlaid on top of the real GRH. * The driver sets the ipoib_vertcflow to 0 if the pseudo-GRH * does not hold valid information. If it is indicated valid, * the driver must additionally provide the sender's qpn in * network byte order in ipoib_sqpn, and not touch the * remaining parts which were DMA'ed in by the IBA hardware. */ typedef struct ipoib_pgrh { uint32_t ipoib_vertcflow; uint32_t ipoib_sqpn; uint32_t ipoib_sgid_pref[2]; uint32_t ipoib_sgid_suff[2]; uint32_t ipoib_dgid_pref[2]; uint32_t ipoib_dgid_suff[2]; } ipoib_pgrh_t; /* * The GRH is also dma'ed into recv buffers, thus space needs * to be allocated for them. */ #define IPOIB_GRH_SIZE sizeof (ipoib_pgrh_t) /* support the RC (reliable connected) mode */ #define IBD_MAC_ADDR_RC 0x80000000 /* support the UC (unreliable connected) mode */ #define IBD_MAC_ADDR_UC 0x40000000 #define IBD_RC_SERVICE_ID 0x100000000000000ULL /* * Legacy OFED had used a wrong service ID (one additional zero digit) for * many years. To interop with legacy OFED, we support this wrong service ID * here. */ #define IBD_RC_SERVICE_ID_OFED_INTEROP 0x1000000000000000ULL #define IBD_RC_MIN_CQ_SIZE 0x7f /* Number of ibt_wc_t provided for each RC channel */ #define IBD_RC_MAX_CQ_WC 0x3f #if defined(_KERNEL) && !defined(_BOOT) #include <sys/ib/ibtl/ibti.h> #include <sys/ib/ib_pkt_hdrs.h> #include <sys/list.h> #include <sys/mac_provider.h> #include <sys/mac_ib.h> #include <sys/modhash.h> /* State of a reliable connected channel (ibd_rc_chan_t->chan_state) */ typedef enum { IBD_RC_STATE_INIT = 0, /* Active side */ IBD_RC_STATE_ACT_REP_RECV, /* reply received */ IBD_RC_STATE_ACT_ESTAB, /* established, ready to send */ IBD_RC_STATE_ACT_REJECT, /* rejected */ /* Someone else is closing this channel, please don't re-close it */ IBD_RC_STATE_ACT_CLOSING, IBD_RC_STATE_ACT_CLOSED, IBD_RC_STATE_ACT_ERROR, /* Passive side */ IBD_RC_STATE_PAS_REQ_RECV, /* request received */ IBD_RC_STATE_PAS_ESTAB, /* established, ready to receive */ IBD_RC_STATE_PAS_REJECT, /* rejected */ IBD_RC_STATE_PAS_CLOSED } ibd_rc_chan_state_t; /* * Structure to encapsulate various types of async requests. */ typedef struct ibd_acache_rq { struct list_node rq_list; /* list of pending work */ int rq_op; /* what operation */ ipoib_mac_t rq_mac; ib_gid_t rq_gid; void *rq_ptr; void *rq_ptr2; } ibd_req_t; typedef struct ibd_mcache { struct list_node mc_list; /* full/non list */ uint8_t mc_jstate; boolean_t mc_fullreap; ibt_mcg_info_t mc_info; ibd_req_t mc_req; /* to queue LEAVE req */ } ibd_mce_t; typedef struct ibd_acache_s { struct list_node ac_list; /* free/active list */ ibt_ud_dest_hdl_t ac_dest; ipoib_mac_t ac_mac; uint32_t ac_ref; ibd_mce_t *ac_mce; /* for MCG AHs */ /* For Reliable Connected mode */ struct ibd_rc_chan_s *ac_chan; /* protect tx_too_big_ongoing */ kmutex_t tx_too_big_mutex; /* Deal with too big packet */ boolean_t tx_too_big_ongoing; } ibd_ace_t; #define IBD_MAX_SQSEG 59 #define IBD_MAX_RQSEG 1 typedef enum { IBD_WQE_SEND, IBD_WQE_RECV } ibd_wqe_type_t; typedef enum { IBD_WQE_TXBUF = 1, IBD_WQE_LSOBUF = 2, IBD_WQE_MAPPED = 3, IBD_WQE_RC_COPYBUF = 4 } ibd_wqe_buftype_t; #ifdef DEBUG typedef struct ibd_rc_stat_s { kstat_named_t rc_rcv_trans_byte; kstat_named_t rc_rcv_trans_pkt; kstat_named_t rc_rcv_copy_byte; kstat_named_t rc_rcv_copy_pkt; kstat_named_t rc_rcv_alloc_fail; kstat_named_t rc_rcq_err; /* fail in rcq handler */ kstat_named_t rc_rwqe_short; /* short rwqe */ kstat_named_t rc_xmt_bytes; /* pkt size <= state->id_rc_tx_copy_thresh */ kstat_named_t rc_xmt_small_pkt; kstat_named_t rc_xmt_fragmented_pkt; /* fail in ibt_map_mem_iov() */ kstat_named_t rc_xmt_map_fail_pkt; /* succ in ibt_map_mem_iov() */ kstat_named_t rc_xmt_map_succ_pkt; kstat_named_t rc_ace_not_found; /* ace not found */ /* no swqe even after recycle */ kstat_named_t rc_scq_no_swqe; /* no tx large buf even after recycle */ kstat_named_t rc_scq_no_largebuf; /* short swqe in ibd_send() */ kstat_named_t rc_swqe_short; /* call mac_tx_update() when there is enough swqe */ kstat_named_t rc_swqe_mac_update; /* short large buf in ibd_send() */ kstat_named_t rc_xmt_buf_short; /* call mac_tx_update() when there is enough Tx large buffers */ kstat_named_t rc_xmt_buf_mac_update; kstat_named_t rc_conn_succ; /* # of success connect */ kstat_named_t rc_conn_fail; /* # of fail connect */ /* ace->ac_chan == NULL for unicast packet */ kstat_named_t rc_null_conn; /* not in active established state */ kstat_named_t rc_no_estab_conn; kstat_named_t rc_act_close; /* call ibd_rc_act_close() */ kstat_named_t rc_pas_close; /* call ibd_rc_pas_close() */ kstat_named_t rc_delay_ace_recycle; kstat_named_t rc_act_close_simultaneous; kstat_named_t rc_reset_cnt; /* # of Reset RC channel */ kstat_named_t rc_timeout_act; kstat_named_t rc_timeout_pas; } ibd_rc_stat_t; #endif typedef struct ibd_rc_chan_list_s { /* This mutex protects chan_list and ibd_rc_chan_t.next */ kmutex_t chan_list_mutex; struct ibd_rc_chan_s *chan_list; } ibd_rc_chan_list_t; typedef struct ibd_rc_tx_largebuf_s { struct ibd_rc_tx_largebuf_s *lb_next; uint8_t *lb_buf; } ibd_rc_tx_largebuf_t; /* * Pre-registered copybuf used for send and receive */ typedef struct ibd_copybuf_s { ibt_wr_ds_t ic_sgl; uint8_t *ic_bufaddr; } ibd_copybuf_t; typedef struct ibd_wqe_s { struct ibd_wqe_s *w_next; ibd_copybuf_t w_copybuf; mblk_t *im_mblk; } ibd_wqe_t; /* * Send WQE */ typedef struct ibd_swqe_s { ibd_wqe_t w_ibd_swqe; ibd_wqe_buftype_t w_buftype; ibt_send_wr_t w_swr; ibd_ace_t *w_ahandle; ibt_mi_hdl_t w_mi_hdl; ibt_wr_ds_t w_sgl[IBD_MAX_SQSEG]; ibd_rc_tx_largebuf_t *w_rc_tx_largebuf; } ibd_swqe_t; #define swqe_next w_ibd_swqe.w_next #define swqe_copybuf w_ibd_swqe.w_copybuf #define swqe_im_mblk w_ibd_swqe.im_mblk #define SWQE_TO_WQE(swqe) (ibd_wqe_t *)&((swqe)->w_ibd_swqe) #define WQE_TO_SWQE(wqe) (ibd_swqe_t *)wqe /* * Receive WQE */ typedef struct ibd_rwqe_s { ibd_wqe_t w_ibd_rwqe; struct ibd_state_s *w_state; ibt_recv_wr_t w_rwr; frtn_t w_freemsg_cb; boolean_t w_freeing_wqe; struct ibd_rc_chan_s *w_chan; } ibd_rwqe_t; #define rwqe_next w_ibd_rwqe.w_next #define rwqe_copybuf w_ibd_rwqe.w_copybuf #define rwqe_im_mblk w_ibd_rwqe.im_mblk #define RWQE_TO_WQE(rwqe) (ibd_wqe_t *)&((rwqe)->w_ibd_rwqe) #define WQE_TO_RWQE(wqe) (ibd_rwqe_t *)wqe typedef struct ibd_list_s { kmutex_t dl_mutex; ibd_wqe_t *dl_head; union { boolean_t pending_sends; uint32_t bufs_outstanding; } ustat; uint32_t dl_cnt; } ibd_list_t; #define dl_pending_sends ustat.pending_sends #define dl_bufs_outstanding ustat.bufs_outstanding /* * LSO buffers * * Under normal circumstances we should never need to use any buffer * that's larger than MTU. Unfortunately, IB HCA has limitations * on the length of SGL that are much smaller than those for regular * ethernet NICs. Since the network layer doesn't care to limit the * number of mblk fragments in any send mp chain, we end up having to * use these larger-than-MTU sized (larger than id_tx_buf_sz actually) * buffers occasionally. */ typedef struct ibd_lsobuf_s { struct ibd_lsobuf_s *lb_next; uint8_t *lb_buf; int lb_isfree; } ibd_lsobuf_t; typedef struct ibd_lsobkt_s { uint8_t *bkt_mem; ibd_lsobuf_t *bkt_bufl; ibd_lsobuf_t *bkt_free_head; ibt_mr_hdl_t bkt_mr_hdl; ibt_mr_desc_t bkt_mr_desc; uint_t bkt_nelem; uint_t bkt_nfree; } ibd_lsobkt_t; #define IBD_PORT_DRIVER 0x1 #define IBD_PARTITION_OBJ 0x2 /* * Posting to a single software rx post queue is contentious, * so break it out to (multiple) an array of queues. * * Try to ensure rx_queue structs fall in different cache lines using a filler. * Note: the RX_QUEUE_CACHE_LINE needs to change if the struct changes. */ #define RX_QUEUE_CACHE_LINE \ (64 - (sizeof (kmutex_t) + sizeof (ibd_wqe_t *) + sizeof (uint_t))) typedef struct ibd_rx_queue_s { kmutex_t rx_post_lock; ibd_wqe_t *rx_head; uint_t rx_cnt; uint8_t rx_pad[RX_QUEUE_CACHE_LINE]; } ibd_rx_queue_t; /* * This structure maintains information per port per HCA * (per network interface). */ typedef struct ibd_state_s { uint_t id_type; dev_info_t *id_dip; ibt_clnt_hdl_t id_ibt_hdl; ibt_hca_hdl_t id_hca_hdl; ibt_pd_hdl_t id_pd_hdl; kmem_cache_t *id_req_kmc; ibd_list_t id_tx_rel_list; uint32_t id_running; uint32_t id_max_sqseg; uint32_t id_max_sqseg_hiwm; ibd_list_t id_tx_list; ddi_softintr_t id_tx; uint32_t id_tx_sends; kmutex_t id_txpost_lock; ibd_swqe_t *id_tx_head; ibd_swqe_t *id_tx_tail; int id_tx_busy; uint_t id_tx_buf_sz; uint8_t *id_tx_bufs; ibd_swqe_t *id_tx_wqes; ibt_mr_hdl_t id_tx_mr_hdl; ibt_mr_desc_t id_tx_mr_desc; kmutex_t id_lso_lock; ibd_lsobkt_t *id_lso; kmutex_t id_scq_poll_lock; int id_scq_poll_busy; ibt_cq_hdl_t id_scq_hdl; ibt_wc_t *id_txwcs; uint32_t id_txwcs_size; int id_rx_nqueues; ibd_rx_queue_t *id_rx_queues; int id_rx_post_queue_index; uint32_t id_rx_post_active; ibd_rwqe_t *id_rx_wqes; uint8_t *id_rx_bufs; ibt_mr_hdl_t id_rx_mr_hdl; ibt_mr_desc_t id_rx_mr_desc; uint_t id_rx_buf_sz; /* * id_ud_num_rwqe * Number of "receive WQE" elements that will be allocated and used * by ibd. This parameter is limited by the maximum channel size of * the HCA. Each buffer in the receive wqe will be of MTU size. */ uint32_t id_ud_num_rwqe; ibd_list_t id_rx_list; ddi_softintr_t id_rx; uint32_t id_rx_bufs_outstanding_limit; uint32_t id_rx_allocb; uint32_t id_rx_allocb_failed; ibd_list_t id_rx_free_list; kmutex_t id_rcq_poll_lock; int id_rcq_poll_busy; uint32_t id_rxwcs_size; ibt_wc_t *id_rxwcs; ibt_cq_hdl_t id_rcq_hdl; ibt_channel_hdl_t id_chnl_hdl; ib_pkey_t id_pkey; uint16_t id_pkix; uint8_t id_port; ibt_mcg_info_t *id_mcinfo; mac_handle_t id_mh; mac_resource_handle_t id_rh; ib_gid_t id_sgid; ib_qpn_t id_qpnum; ipoib_mac_t id_macaddr; ib_gid_t id_mgid; ipoib_mac_t id_bcaddr; int id_mtu; uchar_t id_scope; kmutex_t id_acache_req_lock; kcondvar_t id_acache_req_cv; struct list id_req_list; kt_did_t id_async_thrid; kmutex_t id_ac_mutex; ibd_ace_t *id_ac_hot_ace; struct list id_ah_active; struct list id_ah_free; ipoib_mac_t id_ah_addr; ibd_req_t id_ah_req; char id_ah_op; uint64_t id_ah_error; ibd_ace_t *id_ac_list; mod_hash_t *id_ah_active_hash; kmutex_t id_mc_mutex; struct list id_mc_full; struct list id_mc_non; kmutex_t id_trap_lock; kcondvar_t id_trap_cv; boolean_t id_trap_stop; uint32_t id_trap_inprog; char id_prom_op; kmutex_t id_sched_lock; int id_sched_needed; int id_sched_cnt; int id_sched_lso_cnt; kmutex_t id_link_mutex; link_state_t id_link_state; uint64_t id_link_speed; uint64_t id_num_intrs; uint64_t id_tx_short; /* * id_ud_num_swqe * Number of "send WQE" elements that will be allocated and used by * ibd. When tuning this parameter, the size of pre-allocated, pre- * mapped copy buffer in each of these send wqes must be taken into * account. This copy buffer size is determined by the value of * IBD_TX_BUF_SZ (this is currently set to the same value of * ibd_tx_copy_thresh, but may be changed independently if needed). */ uint32_t id_ud_num_swqe; uint64_t id_xmt_bytes; uint64_t id_rcv_bytes; uint64_t id_multi_xmt; uint64_t id_brd_xmt; uint64_t id_multi_rcv; uint64_t id_brd_rcv; uint64_t id_xmt_pkt; uint64_t id_rcv_pkt; uint32_t id_hwcksum_capab; boolean_t id_lso_policy; boolean_t id_lso_capable; uint_t id_lso_maxlen; int id_hca_res_lkey_capab; ibt_lkey_t id_res_lkey; boolean_t id_bgroup_created; kmutex_t id_macst_lock; kcondvar_t id_macst_cv; uint32_t id_mac_state; /* For Reliable Connected Mode */ boolean_t id_enable_rc; boolean_t rc_enable_srq; int rc_mtu; uint32_t rc_tx_max_sqseg; /* * In IPoIB over Reliable Connected mode, its mac address is added * an "IBD_MAC_ADDR_RC" prefix. But for loopback filter in function * ibd_process_rx(), the input mac address should not include the * "IBD_MAC_ADDR_RC" prefix. * * So, we introduce the rc_macaddr_loopback for the loopback filter in * IPoIB over Reliable Connected mode. * * rc_macaddr_loopback = id_macaddr excludes "IBD_MAC_ADDR_RC" prefix. */ ipoib_mac_t rc_macaddr_loopback; ibt_srv_hdl_t rc_listen_hdl; ibt_sbind_hdl_t rc_listen_bind; ibt_srv_hdl_t rc_listen_hdl_OFED_interop; ibt_sbind_hdl_t rc_listen_bind_OFED_interop; ibd_rc_chan_list_t rc_pass_chan_list; /* obsolete active channel list */ ibd_rc_chan_list_t rc_obs_act_chan_list; kmutex_t rc_ace_recycle_lock; ibd_ace_t *rc_ace_recycle; /* Send */ /* * This mutex protects rc_tx_largebuf_free_head, rc_tx_largebuf_nfree * and ibd_rc_tx_largebuf_t->lb_next */ kmutex_t rc_tx_large_bufs_lock; ibd_rc_tx_largebuf_t *rc_tx_largebuf_free_head; uint_t rc_tx_largebuf_nfree; /* The chunk of whole Tx large buffers */ uint8_t *rc_tx_mr_bufs; ibt_mr_hdl_t rc_tx_mr_hdl; ibt_mr_desc_t rc_tx_mr_desc; ibd_rc_tx_largebuf_t *rc_tx_largebuf_desc_base; /* base addr */ boolean_t rc_enable_iov_map; uint_t rc_max_sqseg_hiwm; /* For SRQ */ uint32_t rc_srq_size; ibt_srq_hdl_t rc_srq_hdl; ibd_list_t rc_srq_rwqe_list; ibd_list_t rc_srq_free_list; ibd_rwqe_t *rc_srq_rwqes; uint8_t *rc_srq_rx_bufs; ibt_mr_hdl_t rc_srq_rx_mr_hdl; ibt_mr_desc_t rc_srq_rx_mr_desc; /* For chained receive */ kmutex_t rc_rx_lock; mblk_t *rc_rx_mp; mblk_t *rc_rx_mp_tail; uint32_t rc_rx_mp_len; uint32_t rc_num_tx_chan; uint32_t rc_num_rx_chan; /* Protect rc_timeout_start and rc_timeout */ kmutex_t rc_timeout_lock; boolean_t rc_timeout_start; timeout_id_t rc_timeout; /* Counters for RC mode */ /* RX */ /* * # of Received packets. These packets are directly transferred to GLD * without copy it */ uint64_t rc_rcv_trans_byte; uint64_t rc_rcv_trans_pkt; /* * # of Received packets. We will allocate new buffers for these packet, * copy their content into new buffers, then transfer to GLD */ uint64_t rc_rcv_copy_byte; uint64_t rc_rcv_copy_pkt; uint64_t rc_rcv_alloc_fail; #ifdef DEBUG uint64_t rc_rwqe_short; /* short rwqe */ #endif /* wc->wc_status != IBT_WC_SUCCESS */ uint64_t rc_rcq_err; /* Tx */ uint64_t rc_xmt_bytes; /* pkt size <= ibd_rc_tx_copy_thresh */ uint64_t rc_xmt_small_pkt; uint64_t rc_xmt_fragmented_pkt; /* fail in ibt_map_mem_iov() */ uint64_t rc_xmt_map_fail_pkt; /* succ in ibt_map_mem_iov() */ uint64_t rc_xmt_map_succ_pkt; uint64_t rc_ace_not_found; uint64_t rc_xmt_drop_too_long_pkt; uint64_t rc_xmt_icmp_too_long_pkt; uint64_t rc_xmt_reenter_too_long_pkt; /* short swqe in ibd_send() */ uint64_t rc_swqe_short; /* call mac_tx_update when there is enough swqe */ uint64_t rc_swqe_mac_update; /* short tx large copy buf in ibd_send() */ uint64_t rc_xmt_buf_short; /* call mac_tx_update when there is enough Tx copy buf */ uint64_t rc_xmt_buf_mac_update; /* No swqe even after call swqe recycle function */ uint64_t rc_scq_no_swqe; /* No large Tx buf even after call swqe recycle function */ uint64_t rc_scq_no_largebuf; /* Connection setup and close */ uint64_t rc_conn_succ; /* time of succ connect */ uint64_t rc_conn_fail; /* time of fail connect */ /* ace->ac_chan == NULL for unicast packet */ uint64_t rc_null_conn; /* not in active established state */ uint64_t rc_no_estab_conn; uint64_t rc_act_close; /* call ibd_rc_act_close() */ uint64_t rc_pas_close; /* call ibd_rc_pas_close() */ uint64_t rc_delay_ace_recycle; uint64_t rc_act_close_simultaneous; /* Fail to close a channel because someone else is still using it */ uint64_t rc_act_close_not_clean; /* RCQ is being invoked when closing RC channel */ uint64_t rc_pas_close_rcq_invoking; /* the counter of reset RC channel */ uint64_t rc_reset_cnt; uint64_t rc_timeout_act; uint64_t rc_timeout_pas; /* * Fail to stop this port because this port is connecting to a remote * port */ uint64_t rc_stop_connect; #ifdef DEBUG kstat_t *rc_ksp; #endif ib_guid_t id_hca_guid; ib_guid_t id_port_guid; datalink_id_t id_dlinkid; datalink_id_t id_plinkid; int id_port_inst; struct ibd_state_s *id_next; boolean_t id_force_create; boolean_t id_bgroup_present; uint_t id_hca_max_chan_sz; /* * UD Mode Tunables * * id_ud_tx_copy_thresh * This sets the threshold at which ibd will attempt to do a bcopy * of the outgoing data into a pre-mapped buffer. IPoIB driver's * send behavior is restricted by various parameters, so setting of * this value must be made after careful considerations only. For * instance, IB HCAs currently impose a relatively small limit * (when compared to ethernet NICs) on the length of the SGL for * transmit. On the other hand, the ip stack could send down mp * chains that are quite long when LSO is enabled. * * id_num_lso_bufs * Number of "larger-than-MTU" copy buffers to use for cases when the * outgoing mblk chain is too fragmented to be used with * ibt_map_mem_iov() and too large to be used with regular MTU-sized * copy buffers. It is not recommended to tune this variable without * understanding the application environment and/or memory resources. * The size of each of these lso buffers is determined by the value of * IBD_LSO_BUFSZ. * * id_num_ah * Number of AH cache entries to allocate * * id_hash_size * Hash table size for the active AH list * */ uint_t id_ud_tx_copy_thresh; uint_t id_num_lso_bufs; uint_t id_num_ah; uint_t id_hash_size; boolean_t id_create_broadcast_group; boolean_t id_allow_coalesce_comp_tuning; uint_t id_ud_rx_comp_count; uint_t id_ud_rx_comp_usec; uint_t id_ud_tx_comp_count; uint_t id_ud_tx_comp_usec; /* RC Mode Tunables */ uint_t id_rc_rx_comp_count; uint_t id_rc_rx_comp_usec; uint_t id_rc_tx_comp_count; uint_t id_rc_tx_comp_usec; /* * id_rc_tx_copy_thresh * This sets the threshold at which ibd will attempt to do a bcopy * of the outgoing data into a pre-mapped buffer. * * id_rc_rx_copy_thresh * If (the size of incoming buffer <= id_rc_rx_copy_thresh), ibd * will attempt to allocate a buffer and do a bcopy of the incoming * data into the allocated buffer. * * id_rc_rx_rwqe_thresh * If (the number of available rwqe < ibd_rc_rx_rwqe_thresh), ibd * will attempt to allocate a buffer and do a bcopy of the incoming * data into the allocated buffer. * * id_rc_num_swqe * 1) Send CQ size = ibd_rc_num_swqe * 2) The send queue size = ibd_rc_num_swqe -1 * 3) Number of pre-allocated Tx buffers for ibt_post_send() = * ibd_rc_num_swqe - 1. * * id_rc_num_rwqe * 1) For non-SRQ, we pre-post id_rc_num_rwqe number of WRs * via ibt_post_receive() for receive queue of each RC channel. * 2) For SRQ and non-SRQ, receive CQ size = id_rc_num_rwqe * * For SRQ * If using SRQ, we allocate id_rc_num_srq number of buffers (the * size of each buffer is equal to RC mtu). And post them by * ibt_post_srq(). * * id_rc_num_srq * id_rc_num_srq should not be larger than id_rc_num_rwqe, * otherwise it will cause a bug with the following warnings: * NOTICE: hermon0: Device Error: EQE cq overrun or protection error * NOTICE: hermon0: Device Error: EQE local work queue catastrophic * error * NOTICE: ibd0: HCA GUID 0003ba0001008984 port 1 PKEY ffff * catastrophic channel error * NOTICE: ibd0: HCA GUID 0003ba0001008984 port 1 PKEY ffff * completion queue error */ uint_t id_rc_tx_copy_thresh; uint_t id_rc_rx_copy_thresh; uint_t id_rc_rx_rwqe_thresh; uint_t id_rc_num_swqe; uint_t id_rc_num_rwqe; uint_t id_rc_num_srq; } ibd_state_t; /* * Structures to track global IBTF data, data that is shared * among the IBD device instances. This includes the one ibt_hdl * and the list of service registrations. */ typedef struct ibd_service_s { struct ibd_service_s *is_link; ibt_srv_hdl_t is_srv_hdl; ib_svc_id_t is_sid; uint_t is_ref_cnt; } ibd_service_t; typedef struct ibd_global_state_s { kmutex_t ig_mutex; ibt_clnt_hdl_t ig_ibt_hdl; uint_t ig_ibt_hdl_ref_cnt; ibd_service_t *ig_service_list; } ibd_global_state_t; typedef struct ibd_rc_msg_hello_s { uint32_t reserved_qpn; uint32_t rx_mtu; } ibd_rc_msg_hello_t; typedef struct ibd_rc_chan_s { struct ibd_rc_chan_s *next; /* channel hdl that we'll be using for Reliable Connected Mode */ ibt_channel_hdl_t chan_hdl; struct ibd_state_s *state; ibd_ace_t *ace; ibd_rc_chan_state_t chan_state; ibd_list_t tx_wqe_list; /* free wqe list */ ibd_list_t tx_rel_list; /* for swqe recycle */ ibd_swqe_t *tx_wqes; /* start address of Tx Buffers */ uint8_t *tx_mr_bufs; ibt_mr_hdl_t tx_mr_hdl; ibt_mr_desc_t tx_mr_desc; ibt_cq_hdl_t scq_hdl; /* Tx completion queue */ ibt_wc_t tx_wc[IBD_RC_MAX_CQ_WC]; ddi_softintr_t scq_softintr; /* For chained send */ kmutex_t tx_post_lock; ibd_swqe_t *tx_head; ibd_swqe_t *tx_tail; int tx_busy; /* For tx buffer recycle */ kmutex_t tx_poll_lock; int tx_poll_busy; /* Rx */ ibd_list_t rx_wqe_list; /* used by ibt_post_recv */ ibd_list_t rx_free_list; /* free rwqe list */ ibt_cq_hdl_t rcq_hdl; /* Rx completion queue */ ibt_wc_t rx_wc[IBD_RC_MAX_CQ_WC]; ibd_rwqe_t *rx_rwqes; /* the chuck of whole rwqes */ uint8_t *rx_bufs; /* the chuck of whole Rx bufs */ ibt_mr_hdl_t rx_mr_hdl; /* ibt_mr_hdl_t for rx_bufs */ ibt_mr_desc_t rx_mr_desc; /* ibt_mr_desc_t for rx_bufs */ /* For chained receive */ kmutex_t rx_lock; mblk_t *rx_mp; mblk_t *rx_mp_tail; uint32_t rx_mp_len; uint32_t rcq_size; uint32_t scq_size; /* * We need two channels for each connection. * One channel for Tx; another channel for Rx. * If "is_tx_chan == B_TRUE", this is a Tx channel. */ boolean_t is_tx_chan; /* * For the connection reaper routine ibd_rc_conn_timeout_call(). * "is_used == B_FALSE" indicates this RC channel has not been used for * a long (=ibd_rc_conn_timeout) time. */ boolean_t is_used; /* * When closing this channel, we need to make sure * "chan->rcq_invoking == 0". */ uint32_t rcq_invoking; } ibd_rc_chan_t; /* * The following functions are defined in "ibd.c". * They are also used by "ibd_cm.c" */ void ibd_print_warn(ibd_state_t *, char *, ...); void ibd_unmap_mem(ibd_state_t *, ibd_swqe_t *); void ibd_queue_work_slot(ibd_state_t *, ibd_req_t *, int); boolean_t ibd_acache_recycle(ibd_state_t *, ipoib_mac_t *, boolean_t); void ibd_dec_ref_ace(ibd_state_t *, ibd_ace_t *); ibd_ace_t *ibd_acache_find(ibd_state_t *, ipoib_mac_t *, boolean_t, int); /* * The following functions are defined in "ibd_cm.c". * They are also used in "ibd.c". */ void ibd_async_rc_process_too_big(ibd_state_t *, ibd_req_t *); void ibd_async_rc_close_act_chan(ibd_state_t *, ibd_req_t *); void ibd_async_rc_recycle_ace(ibd_state_t *, ibd_req_t *); /* Connection Setup/Close Functions */ ibt_status_t ibd_rc_listen(ibd_state_t *); void ibd_rc_stop_listen(ibd_state_t *); ibt_status_t ibd_rc_connect(ibd_state_t *, ibd_ace_t *, ibt_path_info_t *, uint64_t); void ibd_rc_try_connect(ibd_state_t *, ibd_ace_t *, ibt_path_info_t *); void ibd_rc_signal_act_close(ibd_state_t *, ibd_ace_t *); void ibd_rc_signal_ace_recycle(ibd_state_t *, ibd_ace_t *); int ibd_rc_pas_close(ibd_rc_chan_t *, boolean_t, boolean_t); void ibd_rc_close_all_chan(ibd_state_t *); void ibd_rc_conn_timeout_call(void *carg); /* Receive Functions */ int ibd_rc_init_srq_list(ibd_state_t *); void ibd_rc_fini_srq_list(ibd_state_t *); int ibd_rc_repost_srq_free_list(ibd_state_t *); /* Send Functions */ int ibd_rc_init_tx_largebuf_list(ibd_state_t *); void ibd_rc_fini_tx_largebuf_list(ibd_state_t *); ibd_swqe_t *ibd_rc_acquire_swqes(ibd_rc_chan_t *); void ibd_rc_post_send(ibd_rc_chan_t *, ibd_swqe_t *); void ibd_rc_drain_scq(ibd_rc_chan_t *, ibt_cq_hdl_t); void ibd_rc_tx_cleanup(ibd_swqe_t *); /* Others */ void ibd_rc_get_conf(ibd_state_t *); int ibd_rc_init_stats(ibd_state_t *); #endif /* _KERNEL && !_BOOT */ #ifdef __cplusplus } #endif #endif /* _SYS_IB_CLIENTS_IBD_H */
