#define MXGEFW_NDIS
#include "myri10ge_var.h"
#include "rss_eth_z8e.h"
#include "rss_ethp_z8e.h"
#include "mcp_gen_header.h"
#define MYRI10GE_MAX_ETHER_MTU 9014
#define MYRI10GE_MAX_GLD_MTU 9000
#define MYRI10GE_MIN_GLD_MTU 1500
#define MYRI10GE_ETH_STOPPED 0
#define MYRI10GE_ETH_STOPPING 1
#define MYRI10GE_ETH_STARTING 2
#define MYRI10GE_ETH_RUNNING 3
#define MYRI10GE_ETH_OPEN_FAILED 4
#define MYRI10GE_ETH_SUSPENDED_RUNNING 5
static int myri10ge_small_bytes = 510;
static int myri10ge_intr_coal_delay = 125;
static int myri10ge_flow_control = 1;
#if defined __i386 || defined i386 || defined __i386__ || defined __x86_64__
static int myri10ge_nvidia_ecrc_enable = 1;
#endif
static int myri10ge_mtu_override = 0;
static int myri10ge_tx_copylen = 512;
static int myri10ge_deassert_wait = 1;
static int myri10ge_verbose = 0;
static int myri10ge_watchdog_reset = 0;
static int myri10ge_use_msix = 1;
static int myri10ge_max_slices = -1;
static int myri10ge_use_msi = 1;
int myri10ge_force_firmware = 0;
static boolean_t myri10ge_use_lso = B_TRUE;
static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT;
static int myri10ge_tx_hash = 1;
static int myri10ge_lro = 0;
static int myri10ge_lro_cnt = 8;
int myri10ge_lro_max_aggr = 2;
static int myri10ge_lso_copy = 0;
static mblk_t *myri10ge_send_wrapper(void *arg, mblk_t *mp);
int myri10ge_tx_handles_initial = 128;
static kmutex_t myri10ge_param_lock;
static void* myri10ge_db_lastfree;
static int myri10ge_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
static int myri10ge_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
static int myri10ge_quiesce(dev_info_t *dip);
DDI_DEFINE_STREAM_OPS(myri10ge_ops, nulldev, nulldev, myri10ge_attach,
myri10ge_detach, nodev, NULL, D_MP, NULL, myri10ge_quiesce);
static struct modldrv modldrv = {
&mod_driverops,
"Myricom 10G driver (10GbE)",
&myri10ge_ops,
};
static struct modlinkage modlinkage = {
MODREV_1,
{&modldrv, NULL},
};
unsigned char myri10ge_broadcastaddr[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
static ddi_dma_attr_t myri10ge_misc_dma_attr = {
DMA_ATTR_V0,
(uint64_t)0,
(uint64_t)0xffffffffffffffffULL,
(uint64_t)0x7ffffff,
(uint64_t)4096,
(uint_t)0x7f,
(uint32_t)0x1,
(uint64_t)0x7fffffff,
(uint64_t)0x7fffffff,
1,
1,
0
};
static ddi_dma_attr_t myri10ge_rx_jumbo_dma_attr = {
DMA_ATTR_V0,
(uint64_t)0,
(uint64_t)0xffffffffffffffffULL,
(uint64_t)0x7ffffff,
(uint64_t)4096,
(uint_t)0x7f,
(uint32_t)0x1,
(uint64_t)0x7fffffff,
UINT64_MAX,
1,
1,
0
};
static ddi_dma_attr_t myri10ge_rx_std_dma_attr = {
DMA_ATTR_V0,
(uint64_t)0,
(uint64_t)0xffffffffffffffffULL,
(uint64_t)0x7ffffff,
#if defined sparc64 || defined __sparcv9
(uint64_t)4096,
#else
(uint64_t)0x80,
#endif
(uint_t)0x7f,
(uint32_t)0x1,
(uint64_t)0x7fffffff,
#if defined sparc64 || defined __sparcv9
UINT64_MAX,
#else
(uint64_t)0xfff,
#endif
1,
1,
0
};
static ddi_dma_attr_t myri10ge_tx_dma_attr = {
DMA_ATTR_V0,
(uint64_t)0,
(uint64_t)0xffffffffffffffffULL,
(uint64_t)0x7ffffff,
(uint64_t)1,
(uint_t)0x7f,
(uint32_t)0x1,
(uint64_t)0x7fffffff,
UINT64_MAX,
INT32_MAX,
1,
0
};
#if defined sparc64 || defined __sparcv9
#define WC 0
#else
#define WC 1
#endif
struct ddi_device_acc_attr myri10ge_dev_access_attr = {
DDI_DEVICE_ATTR_V0,
DDI_NEVERSWAP_ACC,
#if WC
DDI_MERGING_OK_ACC
#else
DDI_STRICTORDER_ACC
#endif
};
static void myri10ge_watchdog(void *arg);
#ifdef MYRICOM_PRIV
int myri10ge_mtu = MYRI10GE_MAX_ETHER_MTU + MXGEFW_PAD + VLAN_TAGSZ;
#define MYRI10GE_DEFAULT_GLD_MTU MYRI10GE_MAX_GLD_MTU
#else
int myri10ge_mtu = ETHERMAX + MXGEFW_PAD + VLAN_TAGSZ;
#define MYRI10GE_DEFAULT_GLD_MTU MYRI10GE_MIN_GLD_MTU
#endif
int myri10ge_bigbufs_initial = 1024;
int myri10ge_bigbufs_max = 4096;
caddr_t
myri10ge_dma_alloc(dev_info_t *dip, size_t len,
ddi_dma_attr_t *attr, ddi_device_acc_attr_t *accattr,
uint_t alloc_flags, int bind_flags, struct myri10ge_dma_stuff *dma,
int warn, int (*wait)(caddr_t))
{
caddr_t kaddr;
size_t real_length;
ddi_dma_cookie_t cookie;
uint_t count;
int err;
err = ddi_dma_alloc_handle(dip, attr, wait,
NULL, &dma->handle);
if (err != DDI_SUCCESS) {
if (warn)
cmn_err(CE_WARN,
"myri10ge: ddi_dma_alloc_handle failed\n");
goto abort_with_nothing;
}
err = ddi_dma_mem_alloc(dma->handle, len, accattr, alloc_flags,
wait, NULL, &kaddr, &real_length,
&dma->acc_handle);
if (err != DDI_SUCCESS) {
if (warn)
cmn_err(CE_WARN,
"myri10ge: ddi_dma_mem_alloc failed\n");
goto abort_with_handle;
}
err = ddi_dma_addr_bind_handle(dma->handle, NULL, kaddr, len,
bind_flags, wait, NULL, &cookie, &count);
if (err != DDI_SUCCESS) {
if (warn)
cmn_err(CE_WARN,
"myri10ge: ddi_dma_addr_bind_handle failed\n");
goto abort_with_mem;
}
if (count != 1) {
if (warn)
cmn_err(CE_WARN,
"myri10ge: got too many dma segments ");
goto abort_with_bind;
}
dma->low = htonl(MYRI10GE_LOWPART_TO_U32(cookie.dmac_laddress));
dma->high = htonl(MYRI10GE_HIGHPART_TO_U32(cookie.dmac_laddress));
return (kaddr);
abort_with_bind:
(void) ddi_dma_unbind_handle(dma->handle);
abort_with_mem:
ddi_dma_mem_free(&dma->acc_handle);
abort_with_handle:
ddi_dma_free_handle(&dma->handle);
abort_with_nothing:
if (warn) {
cmn_err(CE_WARN, "myri10ge: myri10ge_dma_alloc failed.\n ");
cmn_err(CE_WARN, "args: dip=%p len=0x%lx ddi_dma_attr=%p\n",
(void*) dip, len, (void*) attr);
cmn_err(CE_WARN,
"args: ddi_device_acc_attr=%p alloc_flags=0x%x\n",
(void*) accattr, alloc_flags);
cmn_err(CE_WARN, "args: bind_flags=0x%x dmastuff=%p",
bind_flags, (void*) dma);
}
return (NULL);
}
void
myri10ge_dma_free(struct myri10ge_dma_stuff *dma)
{
(void) ddi_dma_unbind_handle(dma->handle);
ddi_dma_mem_free(&dma->acc_handle);
ddi_dma_free_handle(&dma->handle);
}
static inline void
myri10ge_pio_copy32(void *to, uint32_t *from32, size_t size)
{
register volatile uint32_t *to32;
size_t i;
to32 = (volatile uint32_t *) to;
for (i = (size / 4); i; i--) {
*to32 = *from32;
to32++;
from32++;
}
}
#if defined(_LP64)
static inline void
myri10ge_pio_copy64(void *to, uint64_t *from64, size_t size)
{
register volatile uint64_t *to64;
size_t i;
to64 = (volatile uint64_t *) to;
for (i = (size / 8); i; i--) {
*to64 = *from64;
to64++;
from64++;
}
}
#endif
static inline void
myri10ge_pio_copy(void *to, void *from, size_t size)
{
#if !defined(_LP64)
ASSERT((size % 4) == 0);
myri10ge_pio_copy32(to, (uint32_t *)from, size);
#else
ASSERT((size % 8) == 0);
myri10ge_pio_copy64(to, (uint64_t *)from, size);
#endif
}
static void
myri10ge_jpool_init(struct myri10ge_slice_state *ss)
{
struct myri10ge_jpool_stuff *jpool = &ss->jpool;
bzero(jpool, sizeof (*jpool));
mutex_init(&jpool->mtx, NULL, MUTEX_DRIVER,
ss->mgp->icookie);
jpool->head = NULL;
}
static void
myri10ge_jpool_fini(struct myri10ge_slice_state *ss)
{
struct myri10ge_jpool_stuff *jpool = &ss->jpool;
if (jpool->head != NULL) {
cmn_err(CE_WARN,
"%s: BUG! myri10ge_jpool_fini called on non-empty pool\n",
ss->mgp->name);
}
mutex_destroy(&jpool->mtx);
}
static inline void
myri10ge_submit_8rx(mcp_kreq_ether_recv_t *dst, mcp_kreq_ether_recv_t *src)
{
src->addr_low |= BE_32(1);
myri10ge_pio_copy(dst, src, 4 * sizeof (*src));
mb();
myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src));
mb();
src->addr_low &= ~(BE_32(1));
dst->addr_low = src->addr_low;
mb();
}
static void
myri10ge_pull_jpool(struct myri10ge_slice_state *ss)
{
struct myri10ge_jpool_stuff *jpool = &ss->jpool;
struct myri10ge_jpool_entry *jtail, *j, *jfree;
volatile void *putp;
int i;
jtail = NULL;
if (jpool->head != NULL) {
j = jpool->head;
while (j->next != NULL)
j = j->next;
jtail = j;
}
for (i = 0; i < MYRI10GE_MAX_CPUS; i++) {
putp = &jpool->cpu[i & MYRI10GE_MAX_CPU_MASK].head;
jfree = atomic_swap_ptr(putp, NULL);
if (jfree == NULL)
continue;
if (jtail == NULL) {
jpool->head = jfree;
} else {
jtail->next = jfree;
}
j = jfree;
while (j->next != NULL)
j = j->next;
jtail = j;
}
}
static inline void
myri10ge_restock_jumbos(struct myri10ge_slice_state *ss)
{
struct myri10ge_jpool_stuff *jpool = &ss->jpool;
struct myri10ge_jpool_entry *j;
myri10ge_rx_ring_t *rx;
int i, idx, limit;
rx = &ss->rx_big;
limit = ss->j_rx_cnt + (rx->mask + 1);
for (i = rx->cnt; i != limit; i++) {
idx = i & (rx->mask);
j = jpool->head;
if (j == NULL) {
myri10ge_pull_jpool(ss);
j = jpool->head;
if (j == NULL) {
break;
}
}
jpool->head = j->next;
rx->info[idx].j = j;
rx->shadow[idx].addr_low = j->dma.low;
rx->shadow[idx].addr_high = j->dma.high;
if ((idx & 7) == 7) {
myri10ge_submit_8rx(&rx->lanai[idx - 7],
&rx->shadow[idx - 7]);
}
}
rx->cnt = i;
}
static inline void
myri10ge_unstock_jumbos(struct myri10ge_slice_state *ss)
{
struct myri10ge_jpool_stuff *jpool = &ss->jpool;
struct myri10ge_jpool_entry *j;
myri10ge_rx_ring_t *rx;
int i;
mutex_enter(&jpool->mtx);
rx = &ss->rx_big;
for (i = 0; i < rx->mask + 1; i++) {
j = rx->info[i].j;
rx->info[i].j = NULL;
if (j == NULL)
continue;
j->next = jpool->head;
jpool->head = j;
}
mutex_exit(&jpool->mtx);
}
static void
myri10ge_jfree_rtn(void *arg)
{
struct myri10ge_jpool_entry *j = (struct myri10ge_jpool_entry *)arg;
struct myri10ge_jpool_stuff *jpool;
volatile uintptr_t *putp;
uintptr_t old, new;
jpool = &j->ss->jpool;
putp = (void *)&jpool->cpu[CPU->cpu_seqid & MYRI10GE_MAX_CPU_MASK].head;
new = (uintptr_t)j;
do {
old = *putp;
j->next = (void *)old;
} while (atomic_cas_ulong(putp, old, new) != old);
}
static void
myri10ge_remove_jbuf(struct myri10ge_jpool_entry *j)
{
(void) ddi_dma_unbind_handle(j->dma_handle);
ddi_dma_mem_free(&j->acc_handle);
ddi_dma_free_handle(&j->dma_handle);
kmem_free(j, sizeof (*j));
}
static int
myri10ge_add_jbuf(struct myri10ge_slice_state *ss)
{
struct myri10ge_jpool_entry *j;
struct myri10ge_jpool_stuff *jpool = &ss->jpool;
ddi_dma_attr_t *rx_dma_attr;
size_t real_length;
ddi_dma_cookie_t cookie;
uint_t count;
int err;
if (myri10ge_mtu < 2048)
rx_dma_attr = &myri10ge_rx_std_dma_attr;
else
rx_dma_attr = &myri10ge_rx_jumbo_dma_attr;
again:
j = (struct myri10ge_jpool_entry *)
kmem_alloc(sizeof (*j), KM_SLEEP);
err = ddi_dma_alloc_handle(ss->mgp->dip, rx_dma_attr,
DDI_DMA_DONTWAIT, NULL, &j->dma_handle);
if (err != DDI_SUCCESS)
goto abort_with_j;
err = ddi_dma_mem_alloc(j->dma_handle, myri10ge_mtu,
&myri10ge_dev_access_attr, DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
NULL, &j->buf, &real_length, &j->acc_handle);
if (err != DDI_SUCCESS)
goto abort_with_handle;
err = ddi_dma_addr_bind_handle(j->dma_handle, NULL, j->buf,
real_length, DDI_DMA_READ|DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
NULL, &cookie, &count);
if (err != DDI_SUCCESS)
goto abort_with_mem;
if (rx_dma_attr == &myri10ge_rx_std_dma_attr &&
rx_dma_attr->dma_attr_align != 4096) {
uint32_t start, end;
start = MYRI10GE_LOWPART_TO_U32(cookie.dmac_laddress);
end = start + myri10ge_mtu;
if (((end >> 12) != (start >> 12)) && (start & 4095U)) {
printf("std buffer crossed a 4KB boundary!\n");
myri10ge_remove_jbuf(j);
rx_dma_attr->dma_attr_align = 4096;
rx_dma_attr->dma_attr_seg = UINT64_MAX;
goto again;
}
}
j->dma.low =
htonl(MYRI10GE_LOWPART_TO_U32(cookie.dmac_laddress));
j->dma.high =
htonl(MYRI10GE_HIGHPART_TO_U32(cookie.dmac_laddress));
j->ss = ss;
j->free_func.free_func = myri10ge_jfree_rtn;
j->free_func.free_arg = (char *)j;
mutex_enter(&jpool->mtx);
j->next = jpool->head;
jpool->head = j;
jpool->num_alloc++;
mutex_exit(&jpool->mtx);
return (0);
abort_with_mem:
ddi_dma_mem_free(&j->acc_handle);
abort_with_handle:
ddi_dma_free_handle(&j->dma_handle);
abort_with_j:
kmem_free(j, sizeof (*j));
if (rx_dma_attr == &myri10ge_rx_std_dma_attr &&
rx_dma_attr->dma_attr_align != 4096) {
cmn_err(CE_NOTE,
"!alloc failed, reverting to gran=1\n");
rx_dma_attr->dma_attr_align = 4096;
rx_dma_attr->dma_attr_seg = UINT64_MAX;
goto again;
}
return (err);
}
static int
myri10ge_jfree_cnt(struct myri10ge_jpool_stuff *jpool)
{
int i;
struct myri10ge_jpool_entry *j;
mutex_enter(&jpool->mtx);
j = jpool->head;
i = 0;
while (j != NULL) {
i++;
j = j->next;
}
mutex_exit(&jpool->mtx);
return (i);
}
static int
myri10ge_add_jbufs(struct myri10ge_slice_state *ss, int num, int total)
{
struct myri10ge_jpool_stuff *jpool = &ss->jpool;
int allocated = 0;
int err;
int needed;
if (total && jpool->num_alloc) {
allocated = myri10ge_jfree_cnt(jpool);
needed = num - allocated;
} else {
needed = num;
}
while (needed > 0) {
needed--;
err = myri10ge_add_jbuf(ss);
if (err == 0) {
allocated++;
}
}
return (allocated);
}
static void
myri10ge_remove_jbufs(struct myri10ge_slice_state *ss)
{
struct myri10ge_jpool_stuff *jpool = &ss->jpool;
struct myri10ge_jpool_entry *j;
mutex_enter(&jpool->mtx);
myri10ge_pull_jpool(ss);
while (jpool->head != NULL) {
jpool->num_alloc--;
j = jpool->head;
jpool->head = j->next;
myri10ge_remove_jbuf(j);
}
mutex_exit(&jpool->mtx);
}
static void
myri10ge_carve_up_jbufs_into_small_ring(struct myri10ge_slice_state *ss)
{
struct myri10ge_jpool_stuff *jpool = &ss->jpool;
struct myri10ge_jpool_entry *j = NULL;
caddr_t ptr;
uint32_t dma_low, dma_high;
int idx, len;
unsigned int alloc_size;
dma_low = dma_high = len = 0;
alloc_size = myri10ge_small_bytes + MXGEFW_PAD;
ptr = NULL;
for (idx = 0; idx < ss->rx_small.mask + 1; idx++) {
if (len < alloc_size) {
mutex_enter(&jpool->mtx);
j = jpool->head;
jpool->head = j->next;
j->next = ss->small_jpool;
ss->small_jpool = j;
mutex_exit(&jpool->mtx);
len = myri10ge_mtu;
dma_low = ntohl(j->dma.low);
dma_high = ntohl(j->dma.high);
ptr = j->buf;
}
ss->rx_small.info[idx].ptr = ptr;
ss->rx_small.shadow[idx].addr_low = htonl(dma_low);
ss->rx_small.shadow[idx].addr_high = htonl(dma_high);
len -= alloc_size;
ptr += alloc_size;
dma_low += alloc_size;
}
}
static void
myri10ge_release_small_jbufs(struct myri10ge_slice_state *ss)
{
struct myri10ge_jpool_stuff *jpool = &ss->jpool;
struct myri10ge_jpool_entry *j = NULL;
mutex_enter(&jpool->mtx);
while (ss->small_jpool != NULL) {
j = ss->small_jpool;
ss->small_jpool = j->next;
j->next = jpool->head;
jpool->head = j;
}
mutex_exit(&jpool->mtx);
ss->jbufs_for_smalls = 0;
}
static int
myri10ge_add_tx_handle(struct myri10ge_slice_state *ss)
{
myri10ge_tx_ring_t *tx = &ss->tx;
struct myri10ge_priv *mgp = ss->mgp;
struct myri10ge_tx_dma_handle *handle;
int err;
handle = kmem_zalloc(sizeof (*handle), KM_SLEEP);
err = ddi_dma_alloc_handle(mgp->dip,
&myri10ge_tx_dma_attr,
DDI_DMA_SLEEP, NULL,
&handle->h);
if (err) {
static int limit = 0;
if (limit == 0)
cmn_err(CE_WARN, "%s: Falled to alloc tx dma handle\n",
mgp->name);
limit++;
kmem_free(handle, sizeof (*handle));
return (err);
}
mutex_enter(&tx->handle_lock);
MYRI10GE_SLICE_STAT_INC(tx_handles_alloced);
handle->next = tx->free_tx_handles;
tx->free_tx_handles = handle;
mutex_exit(&tx->handle_lock);
return (DDI_SUCCESS);
}
static void
myri10ge_remove_tx_handles(struct myri10ge_slice_state *ss)
{
myri10ge_tx_ring_t *tx = &ss->tx;
struct myri10ge_tx_dma_handle *handle;
mutex_enter(&tx->handle_lock);
handle = tx->free_tx_handles;
while (handle != NULL) {
tx->free_tx_handles = handle->next;
ddi_dma_free_handle(&handle->h);
kmem_free(handle, sizeof (*handle));
handle = tx->free_tx_handles;
MYRI10GE_SLICE_STAT_DEC(tx_handles_alloced);
}
mutex_exit(&tx->handle_lock);
if (MYRI10GE_SLICE_STAT(tx_handles_alloced) != 0) {
cmn_err(CE_WARN, "%s: %d tx dma handles allocated at close\n",
ss->mgp->name,
(int)MYRI10GE_SLICE_STAT(tx_handles_alloced));
}
}
static void
myri10ge_free_tx_handles(myri10ge_tx_ring_t *tx,
struct myri10ge_tx_dma_handle_head *list)
{
mutex_enter(&tx->handle_lock);
list->tail->next = tx->free_tx_handles;
tx->free_tx_handles = list->head;
mutex_exit(&tx->handle_lock);
}
static void
myri10ge_free_tx_handle_slist(myri10ge_tx_ring_t *tx,
struct myri10ge_tx_dma_handle *handle)
{
struct myri10ge_tx_dma_handle_head list;
if (handle == NULL)
return;
list.head = handle;
list.tail = handle;
while (handle != NULL) {
list.tail = handle;
handle = handle->next;
}
myri10ge_free_tx_handles(tx, &list);
}
static int
myri10ge_alloc_tx_handles(struct myri10ge_slice_state *ss, int count,
struct myri10ge_tx_dma_handle **ret)
{
myri10ge_tx_ring_t *tx = &ss->tx;
struct myri10ge_tx_dma_handle *handle;
int err, i;
mutex_enter(&tx->handle_lock);
for (i = 0; i < count; i++) {
handle = tx->free_tx_handles;
while (handle == NULL) {
mutex_exit(&tx->handle_lock);
err = myri10ge_add_tx_handle(ss);
if (err != DDI_SUCCESS) {
goto abort_with_handles;
}
mutex_enter(&tx->handle_lock);
handle = tx->free_tx_handles;
}
tx->free_tx_handles = handle->next;
handle->next = *ret;
*ret = handle;
}
mutex_exit(&tx->handle_lock);
return (DDI_SUCCESS);
abort_with_handles:
myri10ge_free_tx_handle_slist(tx, *ret);
return (err);
}
static void
myri10ge_unprepare_tx_ring(struct myri10ge_slice_state *ss)
{
myri10ge_tx_ring_t *tx;
struct myri10ge_tx_dma_handle_head handles;
size_t bytes;
int idx;
tx = &ss->tx;
handles.head = NULL;
handles.tail = NULL;
for (idx = 0; idx < ss->tx.mask + 1; idx++) {
if (tx->info[idx].m) {
(void) ddi_dma_unbind_handle(tx->info[idx].handle->h);
handles.head = tx->info[idx].handle;
if (handles.tail == NULL)
handles.tail = tx->info[idx].handle;
freeb(tx->info[idx].m);
tx->info[idx].m = 0;
tx->info[idx].handle = 0;
}
tx->cp[idx].va = NULL;
myri10ge_dma_free(&tx->cp[idx].dma);
}
bytes = sizeof (*tx->cp) * (tx->mask + 1);
kmem_free(tx->cp, bytes);
tx->cp = NULL;
if (handles.head != NULL)
myri10ge_free_tx_handles(tx, &handles);
myri10ge_remove_tx_handles(ss);
}
static inline int
myri10ge_prepare_tx_ring(struct myri10ge_slice_state *ss)
{
struct myri10ge_tx_dma_handle *handles;
int h;
size_t bytes;
bytes = sizeof (*ss->tx.cp) * (ss->tx.mask + 1);
ss->tx.cp = kmem_zalloc(bytes, KM_SLEEP);
if (ss->tx.cp == NULL) {
cmn_err(CE_WARN,
"%s: Failed to allocate tx copyblock storage\n",
ss->mgp->name);
return (DDI_FAILURE);
}
for (h = 0; h < ss->tx.mask + 1; h++) {
ss->tx.cp[h].va = myri10ge_dma_alloc(ss->mgp->dip,
4096, &myri10ge_rx_jumbo_dma_attr,
&myri10ge_dev_access_attr, DDI_DMA_STREAMING,
DDI_DMA_WRITE|DDI_DMA_STREAMING, &ss->tx.cp[h].dma, 1,
DDI_DMA_DONTWAIT);
if (ss->tx.cp[h].va == NULL) {
cmn_err(CE_WARN, "%s: Failed to allocate tx "
"copyblock %d\n", ss->mgp->name, h);
goto abort_with_copyblocks;
}
}
handles = NULL;
(void) myri10ge_alloc_tx_handles(ss, myri10ge_tx_handles_initial,
&handles);
if (handles != NULL)
myri10ge_free_tx_handle_slist(&ss->tx, handles);
return (DDI_SUCCESS);
abort_with_copyblocks:
while (h > 0) {
h--;
myri10ge_dma_free(&ss->tx.cp[h].dma);
}
bytes = sizeof (*ss->tx.cp) * (ss->tx.mask + 1);
kmem_free(ss->tx.cp, bytes);
ss->tx.cp = NULL;
return (DDI_FAILURE);
}
static int
myri10ge_read_mac_addr(struct myri10ge_priv *mgp)
{
#define MYRI10GE_NEXT_STRING(p) while (ptr < limit && *ptr++)
#define myri10ge_digit(c) (((c) >= '0' && (c) <= '9') ? ((c) - '0') : \
(((c) >= 'A' && (c) <= 'F') ? (10 + (c) - 'A') : \
(((c) >= 'a' && (c) <= 'f') ? (10 + (c) - 'a') : -1)))
char *ptr, *limit;
int i, hv, lv;
ptr = mgp->eeprom_strings;
limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE;
while (*ptr != '\0' && ptr < limit) {
if (memcmp(ptr, "MAC=", 4) == 0) {
ptr += 4;
if (myri10ge_verbose)
printf("%s: mac address = %s\n", mgp->name,
ptr);
mgp->mac_addr_string = ptr;
for (i = 0; i < 6; i++) {
if ((ptr + 2) > limit)
goto abort;
if (*(ptr+1) == ':') {
hv = 0;
lv = myri10ge_digit(*ptr); ptr++;
} else {
hv = myri10ge_digit(*ptr); ptr++;
lv = myri10ge_digit(*ptr); ptr++;
}
mgp->mac_addr[i] = (hv << 4) | lv;
ptr++;
}
}
if (memcmp((const void *)ptr, "SN=", 3) == 0) {
ptr += 3;
mgp->sn_str = (char *)ptr;
}
if (memcmp((const void *)ptr, "PC=", 3) == 0) {
ptr += 3;
mgp->pc_str = (char *)ptr;
}
MYRI10GE_NEXT_STRING(ptr);
}
return (0);
abort:
cmn_err(CE_WARN, "%s: failed to parse eeprom_strings", mgp->name);
return (ENXIO);
}
static int
myri10ge_reg_set(dev_info_t *dip, int *reg_set, int *span,
unsigned long *busno, unsigned long *devno,
unsigned long *funcno)
{
#define REGISTER_NUMBER(ip) (ip[0] >> 0 & 0xff)
#define FUNCTION_NUMBER(ip) (ip[0] >> 8 & 0x07)
#define DEVICE_NUMBER(ip) (ip[0] >> 11 & 0x1f)
#define BUS_NUMBER(ip) (ip[0] >> 16 & 0xff)
#define ADDRESS_SPACE(ip) (ip[0] >> 24 & 0x03)
#define PCI_ADDR_HIGH(ip) (ip[1])
#define PCI_ADDR_LOW(ip) (ip[2])
#define PCI_SPAN_HIGH(ip) (ip[3])
#define PCI_SPAN_LOW(ip) (ip[4])
#define MX_DDI_REG_SET_32_BIT_MEMORY_SPACE 2
#define MX_DDI_REG_SET_64_BIT_MEMORY_SPACE 3
int *data, i, *rs;
uint32_t nelementsp;
#ifdef MYRI10GE_REGSET_VERBOSE
char *address_space_name[] = { "Configuration Space",
"I/O Space",
"32-bit Memory Space",
"64-bit Memory Space"
};
#endif
if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
"reg", &data, &nelementsp) != DDI_SUCCESS) {
printf("Could not determine register set.\n");
return (ENXIO);
}
#ifdef MYRI10GE_REGSET_VERBOSE
printf("There are %d register sets.\n", nelementsp / 5);
#endif
if (!nelementsp) {
printf("Didn't find any \"reg\" properties.\n");
ddi_prop_free(data);
return (ENODEV);
}
rs = &data[0];
*busno = BUS_NUMBER(rs);
*devno = DEVICE_NUMBER(rs);
*funcno = FUNCTION_NUMBER(rs);
#ifdef MYRI10GE_REGSET_VERBOSE
printf("*** Scanning for register number.\n");
#endif
for (i = 0; i < nelementsp / 5; i++) {
rs = &data[5 * i];
#ifdef MYRI10GE_REGSET_VERBOSE
printf("Examining register set %d:\n", i);
printf(" Register number = %d.\n", REGISTER_NUMBER(rs));
printf(" Function number = %d.\n", FUNCTION_NUMBER(rs));
printf(" Device number = %d.\n", DEVICE_NUMBER(rs));
printf(" Bus number = %d.\n", BUS_NUMBER(rs));
printf(" Address space = %d (%s ).\n", ADDRESS_SPACE(rs),
address_space_name[ADDRESS_SPACE(rs)]);
printf(" pci address 0x%08x %08x\n", PCI_ADDR_HIGH(rs),
PCI_ADDR_LOW(rs));
printf(" pci span 0x%08x %08x\n", PCI_SPAN_HIGH(rs),
PCI_SPAN_LOW(rs));
#endif
if (ADDRESS_SPACE(rs) == MX_DDI_REG_SET_64_BIT_MEMORY_SPACE ||
ADDRESS_SPACE(rs) == MX_DDI_REG_SET_32_BIT_MEMORY_SPACE) {
*reg_set = i;
#ifdef MYRI10GE_REGSET_VERBOSE
printf("%s uses register set %d.\n",
address_space_name[ADDRESS_SPACE(rs)], *reg_set);
#endif
*span = (PCI_SPAN_LOW(rs));
#ifdef MYRI10GE_REGSET_VERBOSE
printf("Board span is 0x%x\n", *span);
#endif
break;
}
}
ddi_prop_free(data);
if (i >= nelementsp / 5) {
return (EIO);
}
return (0);
}
static int
myri10ge_load_firmware_from_zlib(struct myri10ge_priv *mgp, uint32_t *limit)
{
void *inflate_buffer;
int rv, status;
size_t sram_size = mgp->sram_size - MYRI10GE_EEPROM_STRINGS_SIZE;
size_t destlen;
mcp_gen_header_t *hdr;
unsigned hdr_offset, i;
*limit = 0;
status = 0;
inflate_buffer = kmem_zalloc(sram_size, KM_NOSLEEP);
if (!inflate_buffer) {
cmn_err(CE_WARN,
"%s: Could not allocate buffer to inflate mcp\n",
mgp->name);
return (ENOMEM);
}
destlen = sram_size;
rv = z_uncompress(inflate_buffer, &destlen, mgp->eth_z8e,
mgp->eth_z8e_length);
if (rv != Z_OK) {
cmn_err(CE_WARN, "%s: Could not inflate mcp: %s\n",
mgp->name, z_strerror(rv));
status = ENXIO;
goto abort;
}
*limit = (uint32_t)destlen;
hdr_offset = htonl(*(uint32_t *)(void *)((char *)inflate_buffer +
MCP_HEADER_PTR_OFFSET));
hdr = (void *)((char *)inflate_buffer + hdr_offset);
if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) {
cmn_err(CE_WARN, "%s: Bad firmware type: 0x%x\n", mgp->name,
ntohl(hdr->mcp_type));
status = EIO;
goto abort;
}
(void) strncpy(mgp->fw_version, hdr->version, sizeof (mgp->fw_version));
if (myri10ge_verbose)
printf("%s: firmware id: %s\n", mgp->name, hdr->version);
for (i = 0; i < *limit; i += 256) {
myri10ge_pio_copy((char *)mgp->sram + MYRI10GE_FW_OFFSET + i,
(char *)inflate_buffer + i,
min(256U, (unsigned)(*limit - i)));
mb();
(void) *(int *)(void *)mgp->sram;
mb();
}
abort:
kmem_free(inflate_buffer, sram_size);
return (status);
}
int
myri10ge_send_cmd(struct myri10ge_priv *mgp, uint32_t cmd,
myri10ge_cmd_t *data)
{
mcp_cmd_t *buf;
char buf_bytes[sizeof (*buf) + 8];
volatile mcp_cmd_response_t *response = mgp->cmd;
volatile char *cmd_addr =
(volatile char *)mgp->sram + MXGEFW_ETH_CMD;
int sleep_total = 0;
buf = (mcp_cmd_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
buf->data0 = htonl(data->data0);
buf->data1 = htonl(data->data1);
buf->data2 = htonl(data->data2);
buf->cmd = htonl(cmd);
buf->response_addr.low = mgp->cmd_dma.low;
buf->response_addr.high = mgp->cmd_dma.high;
mutex_enter(&mgp->cmd_lock);
response->result = 0xffffffff;
mb();
myri10ge_pio_copy((void *)cmd_addr, buf, sizeof (*buf));
for (sleep_total = 0; sleep_total < 20; sleep_total++) {
mb();
if (response->result != 0xffffffff) {
if (response->result == 0) {
data->data0 = ntohl(response->data);
mutex_exit(&mgp->cmd_lock);
return (0);
} else if (ntohl(response->result)
== MXGEFW_CMD_UNKNOWN) {
mutex_exit(&mgp->cmd_lock);
return (ENOSYS);
} else if (ntohl(response->result)
== MXGEFW_CMD_ERROR_UNALIGNED) {
mutex_exit(&mgp->cmd_lock);
return (E2BIG);
} else {
cmn_err(CE_WARN,
"%s: command %d failed, result = %d\n",
mgp->name, cmd, ntohl(response->result));
mutex_exit(&mgp->cmd_lock);
return (ENXIO);
}
}
drv_usecwait(1000);
}
mutex_exit(&mgp->cmd_lock);
cmn_err(CE_WARN, "%s: command %d timed out, result = %d\n",
mgp->name, cmd, ntohl(response->result));
return (EAGAIN);
}
static void
myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable)
{
char buf_bytes[72];
volatile uint32_t *confirm;
volatile char *submit;
uint32_t *buf;
int i;
buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
confirm = (volatile uint32_t *)mgp->cmd;
*confirm = 0;
mb();
buf[0] = mgp->cmd_dma.high;
buf[1] = mgp->cmd_dma.low;
buf[2] = htonl(0xffffffff);
buf[3] = htonl(mgp->cmd_dma.high);
buf[4] = htonl(mgp->cmd_dma.low);
buf[5] = htonl(enable);
submit = (volatile char *)(mgp->sram + MXGEFW_BOOT_DUMMY_RDMA);
myri10ge_pio_copy((char *)submit, buf, 64);
mb();
drv_usecwait(1000);
mb();
i = 0;
while (*confirm != 0xffffffff && i < 20) {
drv_usecwait(1000);
i++;
}
if (*confirm != 0xffffffff) {
cmn_err(CE_WARN, "%s: dummy rdma %s failed (%p = 0x%x)",
mgp->name,
(enable ? "enable" : "disable"), (void*) confirm, *confirm);
}
}
static int
myri10ge_load_firmware(struct myri10ge_priv *mgp)
{
myri10ge_cmd_t cmd;
volatile uint32_t *confirm;
volatile char *submit;
char buf_bytes[72];
uint32_t *buf, size;
int status, i;
buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
status = myri10ge_load_firmware_from_zlib(mgp, &size);
if (status) {
cmn_err(CE_WARN, "%s: firmware loading failed\n", mgp->name);
return (status);
}
confirm = (volatile uint32_t *)mgp->cmd;
*confirm = 0;
mb();
buf[0] = mgp->cmd_dma.high;
buf[1] = mgp->cmd_dma.low;
buf[2] = htonl(0xffffffff);
buf[3] = htonl(MYRI10GE_FW_OFFSET + 8);
buf[4] = htonl(size - 8);
buf[5] = htonl(8);
buf[6] = htonl(0);
submit = (volatile char *)(mgp->sram + MXGEFW_BOOT_HANDOFF);
myri10ge_pio_copy((char *)submit, buf, 64);
mb();
drv_usecwait(1000);
mb();
i = 0;
while (*confirm != 0xffffffff && i < 1000) {
drv_usecwait(1000);
i++;
}
if (*confirm != 0xffffffff) {
cmn_err(CE_WARN, "%s: handoff failed (%p = 0x%x)",
mgp->name, (void *) confirm, *confirm);
return (ENXIO);
}
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
if (status != 0) {
cmn_err(CE_WARN, "%s: failed MXGEFW_CMD_GET_RX_RING_SIZE\n",
mgp->name);
return (ENXIO);
}
mgp->max_intr_slots = 2 * (cmd.data0 / sizeof (mcp_dma_addr_t));
myri10ge_dummy_rdma(mgp, 1);
return (0);
}
static int
myri10ge_m_unicst(void *arg, const uint8_t *addr)
{
struct myri10ge_priv *mgp = arg;
myri10ge_cmd_t cmd;
int status;
cmd.data0 = ((addr[0] << 24) | (addr[1] << 16)
| (addr[2] << 8) | addr[3]);
cmd.data1 = ((addr[4] << 8) | (addr[5]));
status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd);
if (status == 0 && (addr != mgp->mac_addr))
(void) memcpy(mgp->mac_addr, addr, sizeof (mgp->mac_addr));
return (status);
}
static int
myri10ge_change_pause(struct myri10ge_priv *mgp, int pause)
{
myri10ge_cmd_t cmd;
int status;
if (pause)
status = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_FLOW_CONTROL,
&cmd);
else
status = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_FLOW_CONTROL,
&cmd);
if (status) {
cmn_err(CE_WARN, "%s: Failed to set flow control mode\n",
mgp->name);
return (ENXIO);
}
mgp->pause = pause;
return (0);
}
static void
myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc)
{
myri10ge_cmd_t cmd;
int status;
if (promisc)
status = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_PROMISC, &cmd);
else
status = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_PROMISC, &cmd);
if (status) {
cmn_err(CE_WARN, "%s: Failed to set promisc mode\n",
mgp->name);
}
}
static int
myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type)
{
myri10ge_cmd_t cmd;
int status;
uint32_t len;
void *dmabench;
struct myri10ge_dma_stuff dmabench_dma;
char *test = " ";
len = mgp->tx_boundary;
dmabench = myri10ge_dma_alloc(mgp->dip, len,
&myri10ge_rx_jumbo_dma_attr, &myri10ge_dev_access_attr,
DDI_DMA_STREAMING, DDI_DMA_RDWR|DDI_DMA_STREAMING,
&dmabench_dma, 1, DDI_DMA_DONTWAIT);
mgp->read_dma = mgp->write_dma = mgp->read_write_dma = 0;
if (dmabench == NULL) {
cmn_err(CE_WARN, "%s dma benchmark aborted\n", mgp->name);
return (ENOMEM);
}
cmd.data0 = ntohl(dmabench_dma.low);
cmd.data1 = ntohl(dmabench_dma.high);
cmd.data2 = len * 0x10000;
status = myri10ge_send_cmd(mgp, test_type, &cmd);
if (status != 0) {
test = "read";
goto abort;
}
mgp->read_dma = ((cmd.data0>>16) * len * 2) / (cmd.data0 & 0xffff);
cmd.data0 = ntohl(dmabench_dma.low);
cmd.data1 = ntohl(dmabench_dma.high);
cmd.data2 = len * 0x1;
status = myri10ge_send_cmd(mgp, test_type, &cmd);
if (status != 0) {
test = "write";
goto abort;
}
mgp->write_dma = ((cmd.data0>>16) * len * 2) / (cmd.data0 & 0xffff);
cmd.data0 = ntohl(dmabench_dma.low);
cmd.data1 = ntohl(dmabench_dma.high);
cmd.data2 = len * 0x10001;
status = myri10ge_send_cmd(mgp, test_type, &cmd);
if (status != 0) {
test = "read/write";
goto abort;
}
mgp->read_write_dma = ((cmd.data0>>16) * len * 2 * 2) /
(cmd.data0 & 0xffff);
abort:
myri10ge_dma_free(&dmabench_dma);
if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST)
cmn_err(CE_WARN, "%s %s dma benchmark failed\n", mgp->name,
test);
return (status);
}
static int
myri10ge_reset(struct myri10ge_priv *mgp)
{
myri10ge_cmd_t cmd;
struct myri10ge_nic_stat *ethstat;
struct myri10ge_slice_state *ss;
int i, status;
size_t bytes;
(void) memset(&cmd, 0, sizeof (cmd));
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd);
if (status != 0) {
cmn_err(CE_WARN, "%s: failed reset\n", mgp->name);
return (ENXIO);
}
bytes = mgp->max_intr_slots * sizeof (*mgp->ss[0].rx_done.entry);
cmd.data0 = (uint32_t)bytes;
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
if (mgp->num_slices > 1) {
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES,
&cmd);
if (status != 0) {
cmn_err(CE_WARN,
"%s: failed to get number of slices\n",
mgp->name);
return (status);
}
cmd.data0 = mgp->num_slices;
cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE |
MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES;
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES,
&cmd);
if (status != 0) {
cmn_err(CE_WARN,
"%s: failed to set number of slices\n",
mgp->name);
return (status);
}
}
for (i = 0; i < mgp->num_slices; i++) {
ss = &mgp->ss[i];
cmd.data0 = ntohl(ss->rx_done.dma.low);
cmd.data1 = ntohl(ss->rx_done.dma.high);
cmd.data2 = i;
status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA,
&cmd);
};
status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd);
for (i = 0; i < mgp->num_slices; i++) {
ss = &mgp->ss[i];
ss->irq_claim = (volatile unsigned int *)
(void *)(mgp->sram + cmd.data0 + 8 * i);
}
if (mgp->ddi_intr_type == DDI_INTR_TYPE_FIXED) {
status |= myri10ge_send_cmd(mgp,
MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, &cmd);
mgp->irq_deassert = (uint32_t *)(void *)(mgp->sram + cmd.data0);
}
status |= myri10ge_send_cmd(mgp,
MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd);
mgp->intr_coal_delay_ptr = (uint32_t *)(void *)(mgp->sram + cmd.data0);
if (status != 0) {
cmn_err(CE_WARN, "%s: failed set interrupt parameters\n",
mgp->name);
return (status);
}
*mgp->intr_coal_delay_ptr = htonl(mgp->intr_coal_delay);
(void) myri10ge_dma_test(mgp, MXGEFW_DMA_TEST);
for (i = 0; i < mgp->num_slices; i++) {
ss = &mgp->ss[i];
bytes = mgp->max_intr_slots *
sizeof (*mgp->ss[0].rx_done.entry);
(void) memset(ss->rx_done.entry, 0, bytes);
ss->tx.req = 0;
ss->tx.done = 0;
ss->tx.pkt_done = 0;
ss->rx_big.cnt = 0;
ss->rx_small.cnt = 0;
ss->rx_done.idx = 0;
ss->rx_done.cnt = 0;
ss->rx_token = 0;
ss->tx.watchdog_done = 0;
ss->tx.watchdog_req = 0;
ss->tx.active = 0;
ss->tx.activate = 0;
}
mgp->watchdog_rx_pause = 0;
if (mgp->ksp_stat != NULL) {
ethstat = (struct myri10ge_nic_stat *)mgp->ksp_stat->ks_data;
ethstat->link_changes.value.ul = 0;
}
status = myri10ge_m_unicst(mgp, mgp->mac_addr);
myri10ge_change_promisc(mgp, 0);
(void) myri10ge_change_pause(mgp, mgp->pause);
return (status);
}
static int
myri10ge_init_toeplitz(struct myri10ge_priv *mgp)
{
myri10ge_cmd_t cmd;
int i, b, s, t, j;
int status;
uint32_t k[8];
uint32_t tmp;
uint8_t *key;
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RSS_KEY_OFFSET,
&cmd);
if (status != 0) {
cmn_err(CE_WARN, "%s: failed to get rss key\n",
mgp->name);
return (EIO);
}
myri10ge_pio_copy32(mgp->rss_key,
(uint32_t *)(void*)((char *)mgp->sram + cmd.data0),
sizeof (mgp->rss_key));
mgp->toeplitz_hash_table = kmem_alloc(sizeof (uint32_t) * 12 * 256,
KM_SLEEP);
key = (uint8_t *)mgp->rss_key;
t = 0;
for (b = 0; b < 12; b++) {
for (s = 0; s < 8; s++) {
k[s] = 0;
for (j = 0; j < 32; j++) {
int bit = b*8+s+j;
bit = 0x1 & (key[bit / 8] >> (7 -(bit & 0x7)));
k[s] |= bit << (31 - j);
}
}
for (i = 0; i <= 0xff; i++) {
tmp = 0;
if (i & (1 << 7)) { tmp ^= k[0]; }
if (i & (1 << 6)) { tmp ^= k[1]; }
if (i & (1 << 5)) { tmp ^= k[2]; }
if (i & (1 << 4)) { tmp ^= k[3]; }
if (i & (1 << 3)) { tmp ^= k[4]; }
if (i & (1 << 2)) { tmp ^= k[5]; }
if (i & (1 << 1)) { tmp ^= k[6]; }
if (i & (1 << 0)) { tmp ^= k[7]; }
mgp->toeplitz_hash_table[t++] = tmp;
}
}
return (0);
}
static inline struct myri10ge_slice_state *
myri10ge_toeplitz_send_hash(struct myri10ge_priv *mgp, struct ip *ip)
{
struct tcphdr *hdr;
uint32_t saddr, daddr;
uint32_t hash, slice;
uint32_t *table = mgp->toeplitz_hash_table;
uint16_t src, dst;
saddr = ntohl(ip->ip_src.s_addr);
daddr = ntohl(ip->ip_dst.s_addr);
hash = table[(256 * 0) + ((daddr >> 24) & 0xff)];
hash ^= table[(256 * 1) + ((daddr >> 16) & 0xff)];
hash ^= table[(256 * 2) + ((daddr >> 8) & 0xff)];
hash ^= table[(256 * 3) + ((daddr) & 0xff)];
hash ^= table[(256 * 4) + ((saddr >> 24) & 0xff)];
hash ^= table[(256 * 5) + ((saddr >> 16) & 0xff)];
hash ^= table[(256 * 6) + ((saddr >> 8) & 0xff)];
hash ^= table[(256 * 7) + ((saddr) & 0xff)];
if ((myri10ge_rss_hash & MXGEFW_RSS_HASH_TYPE_TCP_IPV4) &&
ip->ip_p == IPPROTO_TCP) {
hdr = (struct tcphdr *)(void *)
(((uint8_t *)ip) + (ip->ip_hl << 2));
src = ntohs(hdr->th_sport);
dst = ntohs(hdr->th_dport);
hash ^= table[(256 * 8) + ((dst >> 8) & 0xff)];
hash ^= table[(256 * 9) + ((dst) & 0xff)];
hash ^= table[(256 * 10) + ((src >> 8) & 0xff)];
hash ^= table[(256 * 11) + ((src) & 0xff)];
}
slice = (mgp->num_slices - 1) & hash;
return (&mgp->ss[slice]);
}
static inline struct myri10ge_slice_state *
myri10ge_simple_send_hash(struct myri10ge_priv *mgp, struct ip *ip)
{
struct tcphdr *hdr;
uint32_t slice, hash_val;
if (ip->ip_p != IPPROTO_TCP && ip->ip_p != IPPROTO_UDP) {
return (&mgp->ss[0]);
}
hdr = (struct tcphdr *)(void *)(((uint8_t *)ip) + (ip->ip_hl << 2));
hash_val = ntohs(hdr->th_dport) & 0xff;
if (myri10ge_rss_hash == MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT)
hash_val += ntohs(hdr->th_sport) & 0xff;
slice = (mgp->num_slices - 1) & hash_val;
return (&mgp->ss[slice]);
}
static inline struct myri10ge_slice_state *
myri10ge_send_hash(struct myri10ge_priv *mgp, mblk_t *mp)
{
unsigned int slice = 0;
struct ether_header *eh;
struct ether_vlan_header *vh;
struct ip *ip;
int ehl, ihl;
if (mgp->num_slices == 1)
return (&mgp->ss[0]);
if (myri10ge_tx_hash == 0) {
slice = CPU->cpu_id & (mgp->num_slices - 1);
return (&mgp->ss[slice]);
}
ehl = sizeof (*eh);
ihl = sizeof (*ip);
if ((MBLKL(mp)) < (ehl + ihl + 8))
return (&mgp->ss[0]);
eh = (struct ether_header *)(void *)mp->b_rptr;
ip = (struct ip *)(void *)(eh + 1);
if (eh->ether_type != BE_16(ETHERTYPE_IP)) {
if (eh->ether_type != BE_16(ETHERTYPE_VLAN))
return (&mgp->ss[0]);
vh = (struct ether_vlan_header *)(void *)mp->b_rptr;
if (vh->ether_type != BE_16(ETHERTYPE_IP))
return (&mgp->ss[0]);
ehl += 4;
ip = (struct ip *)(void *)(vh + 1);
}
ihl = ip->ip_hl << 2;
if (MBLKL(mp) < (ehl + ihl + 8))
return (&mgp->ss[0]);
switch (myri10ge_rss_hash) {
case MXGEFW_RSS_HASH_TYPE_IPV4:
case MXGEFW_RSS_HASH_TYPE_TCP_IPV4:
case (MXGEFW_RSS_HASH_TYPE_IPV4|MXGEFW_RSS_HASH_TYPE_TCP_IPV4):
return (myri10ge_toeplitz_send_hash(mgp, ip));
case MXGEFW_RSS_HASH_TYPE_SRC_PORT:
case MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT:
return (myri10ge_simple_send_hash(mgp, ip));
default:
break;
}
return (&mgp->ss[0]);
}
static int
myri10ge_setup_slice(struct myri10ge_slice_state *ss)
{
struct myri10ge_priv *mgp = ss->mgp;
myri10ge_cmd_t cmd;
int tx_ring_size, rx_ring_size;
int tx_ring_entries, rx_ring_entries;
int slice, status;
int allocated, idx;
size_t bytes;
slice = ss - mgp->ss;
cmd.data0 = slice;
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd);
tx_ring_size = cmd.data0;
cmd.data0 = slice;
status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
if (status != 0)
return (status);
rx_ring_size = cmd.data0;
tx_ring_entries = tx_ring_size / sizeof (struct mcp_kreq_ether_send);
rx_ring_entries = rx_ring_size / sizeof (struct mcp_dma_addr);
ss->tx.mask = tx_ring_entries - 1;
ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1;
cmd.data0 = slice;
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, &cmd);
ss->tx.lanai = (mcp_kreq_ether_send_t *)(void *)(mgp->sram + cmd.data0);
if (mgp->num_slices > 1) {
ss->tx.go = (char *)mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice;
ss->tx.stop = (char *)mgp->sram + MXGEFW_ETH_SEND_STOP +
64 * slice;
} else {
ss->tx.go = NULL;
ss->tx.stop = NULL;
}
cmd.data0 = slice;
status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd);
ss->rx_small.lanai = (mcp_kreq_ether_recv_t *)
(void *)(mgp->sram + cmd.data0);
cmd.data0 = slice;
status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd);
ss->rx_big.lanai = (mcp_kreq_ether_recv_t *)(void *)
(mgp->sram + cmd.data0);
if (status != 0) {
cmn_err(CE_WARN,
"%s: failed to get ring sizes or locations\n", mgp->name);
return (status);
}
status = ENOMEM;
bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow);
ss->rx_small.shadow = kmem_zalloc(bytes, KM_SLEEP);
if (ss->rx_small.shadow == NULL)
goto abort;
(void) memset(ss->rx_small.shadow, 0, bytes);
bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow);
ss->rx_big.shadow = kmem_zalloc(bytes, KM_SLEEP);
if (ss->rx_big.shadow == NULL)
goto abort_with_rx_small_shadow;
(void) memset(ss->rx_big.shadow, 0, bytes);
bytes = tx_ring_entries * sizeof (*ss->tx.info);
ss->tx.info = kmem_zalloc(bytes, KM_SLEEP);
if (ss->tx.info == NULL)
goto abort_with_rx_big_shadow;
(void) memset(ss->tx.info, 0, bytes);
bytes = rx_ring_entries * sizeof (*ss->rx_small.info);
ss->rx_small.info = kmem_zalloc(bytes, KM_SLEEP);
if (ss->rx_small.info == NULL)
goto abort_with_tx_info;
(void) memset(ss->rx_small.info, 0, bytes);
bytes = rx_ring_entries * sizeof (*ss->rx_big.info);
ss->rx_big.info = kmem_zalloc(bytes, KM_SLEEP);
if (ss->rx_big.info == NULL)
goto abort_with_rx_small_info;
(void) memset(ss->rx_big.info, 0, bytes);
ss->tx.stall = ss->tx.sched = 0;
ss->tx.stall_early = ss->tx.stall_late = 0;
ss->jbufs_for_smalls = 1 + (1 + ss->rx_small.mask) /
(myri10ge_mtu / (myri10ge_small_bytes + MXGEFW_PAD));
allocated = myri10ge_add_jbufs(ss,
myri10ge_bigbufs_initial + ss->jbufs_for_smalls, 1);
if (allocated < ss->jbufs_for_smalls + myri10ge_bigbufs_initial) {
cmn_err(CE_WARN,
"%s: Could not allocate enough receive buffers (%d/%d)\n",
mgp->name, allocated,
myri10ge_bigbufs_initial + ss->jbufs_for_smalls);
goto abort_with_jumbos;
}
myri10ge_carve_up_jbufs_into_small_ring(ss);
ss->j_rx_cnt = 0;
mutex_enter(&ss->jpool.mtx);
if (allocated < rx_ring_entries)
ss->jpool.low_water = allocated / 4;
else
ss->jpool.low_water = rx_ring_entries / 2;
(void) memset(ss->rx_big.shadow, 1,
(ss->rx_big.mask + 1) * sizeof (ss->rx_big.shadow[0]));
for (idx = 7; idx <= ss->rx_big.mask; idx += 8) {
myri10ge_submit_8rx(&ss->rx_big.lanai[idx - 7],
&ss->rx_big.shadow[idx - 7]);
mb();
}
myri10ge_restock_jumbos(ss);
for (idx = 7; idx <= ss->rx_small.mask; idx += 8) {
myri10ge_submit_8rx(&ss->rx_small.lanai[idx - 7],
&ss->rx_small.shadow[idx - 7]);
mb();
}
ss->rx_small.cnt = ss->rx_small.mask + 1;
mutex_exit(&ss->jpool.mtx);
status = myri10ge_prepare_tx_ring(ss);
if (status != 0)
goto abort_with_small_jbufs;
cmd.data0 = ntohl(ss->fw_stats_dma.low);
cmd.data1 = ntohl(ss->fw_stats_dma.high);
cmd.data2 = sizeof (mcp_irq_data_t);
cmd.data2 |= (slice << 16);
bzero(ss->fw_stats, sizeof (*ss->fw_stats));
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd);
if (status == ENOSYS) {
cmd.data0 = ntohl(ss->fw_stats_dma.low) +
offsetof(mcp_irq_data_t, send_done_count);
cmd.data1 = ntohl(ss->fw_stats_dma.high);
status = myri10ge_send_cmd(mgp,
MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, &cmd);
}
if (status) {
cmn_err(CE_WARN, "%s: Couldn't set stats DMA\n", mgp->name);
goto abort_with_tx;
}
return (0);
abort_with_tx:
myri10ge_unprepare_tx_ring(ss);
abort_with_small_jbufs:
myri10ge_release_small_jbufs(ss);
abort_with_jumbos:
if (allocated != 0) {
mutex_enter(&ss->jpool.mtx);
ss->jpool.low_water = 0;
mutex_exit(&ss->jpool.mtx);
myri10ge_unstock_jumbos(ss);
myri10ge_remove_jbufs(ss);
}
bytes = rx_ring_entries * sizeof (*ss->rx_big.info);
kmem_free(ss->rx_big.info, bytes);
abort_with_rx_small_info:
bytes = rx_ring_entries * sizeof (*ss->rx_small.info);
kmem_free(ss->rx_small.info, bytes);
abort_with_tx_info:
bytes = tx_ring_entries * sizeof (*ss->tx.info);
kmem_free(ss->tx.info, bytes);
abort_with_rx_big_shadow:
bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow);
kmem_free(ss->rx_big.shadow, bytes);
abort_with_rx_small_shadow:
bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow);
kmem_free(ss->rx_small.shadow, bytes);
abort:
return (status);
}
static void
myri10ge_teardown_slice(struct myri10ge_slice_state *ss)
{
int tx_ring_entries, rx_ring_entries;
size_t bytes;
if (ss->tx.cp == NULL)
return;
myri10ge_unprepare_tx_ring(ss);
mutex_enter(&ss->jpool.mtx);
ss->jpool.low_water = 0;
mutex_exit(&ss->jpool.mtx);
myri10ge_release_small_jbufs(ss);
myri10ge_unstock_jumbos(ss);
myri10ge_remove_jbufs(ss);
rx_ring_entries = ss->rx_big.mask + 1;
tx_ring_entries = ss->tx.mask + 1;
bytes = rx_ring_entries * sizeof (*ss->rx_big.info);
kmem_free(ss->rx_big.info, bytes);
bytes = rx_ring_entries * sizeof (*ss->rx_small.info);
kmem_free(ss->rx_small.info, bytes);
bytes = tx_ring_entries * sizeof (*ss->tx.info);
kmem_free(ss->tx.info, bytes);
bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow);
kmem_free(ss->rx_big.shadow, bytes);
bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow);
kmem_free(ss->rx_small.shadow, bytes);
}
static int
myri10ge_start_locked(struct myri10ge_priv *mgp)
{
myri10ge_cmd_t cmd;
int status, big_pow2, i;
volatile uint8_t *itable;
status = DDI_SUCCESS;
status = myri10ge_reset(mgp);
if (status != 0) {
cmn_err(CE_WARN, "%s: failed reset\n", mgp->name);
return (DDI_FAILURE);
}
if (mgp->num_slices > 1) {
cmd.data0 = mgp->num_slices;
cmd.data1 = 1;
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES,
&cmd);
if (status != 0) {
cmn_err(CE_WARN,
"%s: failed to set number of slices\n",
mgp->name);
goto abort_with_nothing;
}
cmd.data0 = mgp->num_slices;
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE,
&cmd);
status |= myri10ge_send_cmd(mgp,
MXGEFW_CMD_GET_RSS_TABLE_OFFSET, &cmd);
if (status != 0) {
cmn_err(CE_WARN,
"%s: failed to setup rss tables\n", mgp->name);
}
itable = mgp->sram + cmd.data0;
for (i = 0; i < mgp->num_slices; i++)
itable[i] = (uint8_t)i;
if (myri10ge_rss_hash & MYRI10GE_TOEPLITZ_HASH) {
status = myri10ge_init_toeplitz(mgp);
if (status != 0) {
cmn_err(CE_WARN, "%s: failed to setup "
"toeplitz tx hash table", mgp->name);
goto abort_with_nothing;
}
}
cmd.data0 = 1;
cmd.data1 = myri10ge_rss_hash;
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE,
&cmd);
if (status != 0) {
cmn_err(CE_WARN,
"%s: failed to enable slices\n", mgp->name);
goto abort_with_toeplitz;
}
}
for (i = 0; i < mgp->num_slices; i++) {
status = myri10ge_setup_slice(&mgp->ss[i]);
if (status != 0)
goto abort_with_slices;
}
big_pow2 = myri10ge_mtu + MXGEFW_PAD;
while (!ISP2(big_pow2))
big_pow2++;
cmd.data0 = myri10ge_mtu;
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd);
cmd.data0 = myri10ge_small_bytes;
status |=
myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd);
cmd.data0 = big_pow2;
status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd);
if (status) {
cmn_err(CE_WARN, "%s: Couldn't set buffer sizes\n", mgp->name);
goto abort_with_slices;
}
cmd.data0 = 1;
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd);
if (status) {
cmn_err(CE_WARN, "%s: unable to setup TSO (%d)\n",
mgp->name, status);
} else {
mgp->features |= MYRI10GE_TSO;
}
mgp->link_state = -1;
mgp->rdma_tags_available = 15;
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd);
if (status) {
cmn_err(CE_WARN, "%s: unable to start ethernet\n", mgp->name);
goto abort_with_slices;
}
mgp->running = MYRI10GE_ETH_RUNNING;
return (DDI_SUCCESS);
abort_with_slices:
for (i = 0; i < mgp->num_slices; i++)
myri10ge_teardown_slice(&mgp->ss[i]);
mgp->running = MYRI10GE_ETH_STOPPED;
abort_with_toeplitz:
if (mgp->toeplitz_hash_table != NULL) {
kmem_free(mgp->toeplitz_hash_table,
sizeof (uint32_t) * 12 * 256);
mgp->toeplitz_hash_table = NULL;
}
abort_with_nothing:
return (DDI_FAILURE);
}
static void
myri10ge_stop_locked(struct myri10ge_priv *mgp)
{
int status, old_down_cnt;
myri10ge_cmd_t cmd;
int wait_time = 10;
int i, polling;
old_down_cnt = mgp->down_cnt;
mb();
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd);
if (status) {
cmn_err(CE_WARN, "%s: Couldn't bring down link\n", mgp->name);
}
while (old_down_cnt == *((volatile int *)&mgp->down_cnt)) {
delay(1 * drv_usectohz(1000000));
wait_time--;
if (wait_time == 0)
break;
}
again:
if (old_down_cnt == *((volatile int *)&mgp->down_cnt)) {
cmn_err(CE_WARN, "%s: didn't get down irq\n", mgp->name);
for (i = 0; i < mgp->num_slices; i++) {
mutex_enter(&mgp->ss[i].rx_lock);
printf("%s: slice %d rx irq idle\n",
mgp->name, i);
mutex_exit(&mgp->ss[i].rx_lock);
mutex_enter(&mgp->ss->poll_lock);
polling = mgp->ss->rx_polling;
mutex_exit(&mgp->ss->poll_lock);
if (polling) {
printf("%s: slice %d is polling\n",
mgp->name, i);
delay(1 * drv_usectohz(1000000));
goto again;
}
}
delay(1 * drv_usectohz(1000000));
if (old_down_cnt == *((volatile int *)&mgp->down_cnt)) {
cmn_err(CE_WARN, "%s: Never got down irq\n", mgp->name);
}
}
for (i = 0; i < mgp->num_slices; i++)
myri10ge_teardown_slice(&mgp->ss[i]);
if (mgp->toeplitz_hash_table != NULL) {
kmem_free(mgp->toeplitz_hash_table,
sizeof (uint32_t) * 12 * 256);
mgp->toeplitz_hash_table = NULL;
}
mgp->running = MYRI10GE_ETH_STOPPED;
}
static int
myri10ge_m_start(void *arg)
{
struct myri10ge_priv *mgp = arg;
int status;
mutex_enter(&mgp->intrlock);
if (mgp->running != MYRI10GE_ETH_STOPPED) {
mutex_exit(&mgp->intrlock);
return (DDI_FAILURE);
}
status = myri10ge_start_locked(mgp);
mutex_exit(&mgp->intrlock);
if (status != DDI_SUCCESS)
return (status);
mgp->timer_id = timeout(myri10ge_watchdog, mgp,
mgp->timer_ticks);
return (DDI_SUCCESS);
}
static void
myri10ge_m_stop(void *arg)
{
struct myri10ge_priv *mgp = arg;
mutex_enter(&mgp->intrlock);
if (mgp->running != MYRI10GE_ETH_RUNNING) {
mutex_exit(&mgp->intrlock);
return;
}
mgp->running = MYRI10GE_ETH_STOPPING;
mutex_exit(&mgp->intrlock);
(void) untimeout(mgp->timer_id);
mutex_enter(&mgp->intrlock);
myri10ge_stop_locked(mgp);
mutex_exit(&mgp->intrlock);
}
static inline void
myri10ge_rx_csum(mblk_t *mp, struct myri10ge_rx_ring_stats *s, uint32_t csum)
{
struct ether_header *eh;
struct ip *ip;
struct ip6_hdr *ip6;
uint32_t start, stuff, end, partial, hdrlen;
csum = ntohs((uint16_t)csum);
eh = (struct ether_header *)(void *)mp->b_rptr;
hdrlen = sizeof (*eh);
if (eh->ether_dhost.ether_addr_octet[0] & 1) {
if (0 == (bcmp(eh->ether_dhost.ether_addr_octet,
myri10ge_broadcastaddr, sizeof (eh->ether_dhost))))
s->brdcstrcv++;
else
s->multircv++;
}
if (eh->ether_type == BE_16(ETHERTYPE_VLAN)) {
partial = *(uint32_t *)
(void *)(mp->b_rptr + ETHERNET_HEADER_SIZE);
csum += ~partial;
csum += (csum < ~partial);
csum = (csum >> 16) + (csum & 0xFFFF);
csum = (csum >> 16) + (csum & 0xFFFF);
hdrlen += VLAN_TAGSZ;
}
if (eh->ether_type == BE_16(ETHERTYPE_IP)) {
ip = (struct ip *)(void *)(mp->b_rptr + hdrlen);
start = ip->ip_hl << 2;
if (ip->ip_p == IPPROTO_TCP)
stuff = start + offsetof(struct tcphdr, th_sum);
else if (ip->ip_p == IPPROTO_UDP)
stuff = start + offsetof(struct udphdr, uh_sum);
else
return;
end = ntohs(ip->ip_len);
} else if (eh->ether_type == BE_16(ETHERTYPE_IPV6)) {
ip6 = (struct ip6_hdr *)(void *)(mp->b_rptr + hdrlen);
start = sizeof (*ip6);
if (ip6->ip6_nxt == IPPROTO_TCP) {
stuff = start + offsetof(struct tcphdr, th_sum);
} else if (ip6->ip6_nxt == IPPROTO_UDP)
stuff = start + offsetof(struct udphdr, uh_sum);
else
return;
end = start + ntohs(ip6->ip6_plen);
partial = myri10ge_csum_generic((uint16_t *)ip6, sizeof (*ip6));
csum += ~partial;
csum += (csum < ~partial);
csum = (csum >> 16) + (csum & 0xFFFF);
csum = (csum >> 16) + (csum & 0xFFFF);
} else {
return;
}
if (MBLKL(mp) > hdrlen + end) {
return;
}
mac_hcksum_set(mp, start, stuff, end, csum, HCK_PARTIALCKSUM);
}
static mblk_t *
myri10ge_rx_done_small(struct myri10ge_slice_state *ss, uint32_t len,
uint32_t csum)
{
mblk_t *mp;
myri10ge_rx_ring_t *rx;
int idx;
rx = &ss->rx_small;
idx = rx->cnt & rx->mask;
ss->rx_small.cnt++;
mp = allocb(len + MXGEFW_PAD, 0);
if (mp == NULL) {
MYRI10GE_ATOMIC_SLICE_STAT_INC(rx_small_nobuf);
goto abort;
}
bcopy(ss->rx_small.info[idx].ptr,
(caddr_t)mp->b_wptr, len + MXGEFW_PAD);
mp->b_wptr += len + MXGEFW_PAD;
mp->b_rptr += MXGEFW_PAD;
ss->rx_stats.ibytes += len;
ss->rx_stats.ipackets += 1;
myri10ge_rx_csum(mp, &ss->rx_stats, csum);
abort:
if ((idx & 7) == 7) {
myri10ge_submit_8rx(&rx->lanai[idx - 7],
&rx->shadow[idx - 7]);
}
return (mp);
}
static mblk_t *
myri10ge_rx_done_big(struct myri10ge_slice_state *ss, uint32_t len,
uint32_t csum)
{
struct myri10ge_jpool_stuff *jpool;
struct myri10ge_jpool_entry *j;
mblk_t *mp;
int idx, num_owned_by_mcp;
jpool = &ss->jpool;
idx = ss->j_rx_cnt & ss->rx_big.mask;
j = ss->rx_big.info[idx].j;
if (j == NULL) {
printf("%s: null j at idx=%d, rx_big.cnt = %d, j_rx_cnt=%d\n",
ss->mgp->name, idx, ss->rx_big.cnt, ss->j_rx_cnt);
return (NULL);
}
ss->rx_big.info[idx].j = NULL;
ss->j_rx_cnt++;
num_owned_by_mcp = ss->rx_big.cnt - ss->j_rx_cnt;
if (num_owned_by_mcp < jpool->low_water) {
mutex_enter(&jpool->mtx);
myri10ge_restock_jumbos(ss);
mutex_exit(&jpool->mtx);
num_owned_by_mcp = ss->rx_big.cnt - ss->j_rx_cnt;
if (num_owned_by_mcp < 16) {
MYRI10GE_ATOMIC_SLICE_STAT_INC(rx_copy);
mp = allocb(len + MXGEFW_PAD, 0);
if (mp == NULL) {
goto abort;
}
bcopy(j->buf,
(caddr_t)mp->b_wptr, len + MXGEFW_PAD);
myri10ge_jfree_rtn(j);
mutex_enter(&jpool->mtx);
myri10ge_restock_jumbos(ss);
mutex_exit(&jpool->mtx);
goto set_len;
}
}
mp = desballoc((unsigned char *)j->buf, myri10ge_mtu, 0, &j->free_func);
if (mp == NULL) {
goto abort;
}
set_len:
mp->b_rptr += MXGEFW_PAD;
mp->b_wptr = ((unsigned char *) mp->b_rptr + len);
ss->rx_stats.ibytes += len;
ss->rx_stats.ipackets += 1;
myri10ge_rx_csum(mp, &ss->rx_stats, csum);
return (mp);
abort:
myri10ge_jfree_rtn(j);
MYRI10GE_ATOMIC_SLICE_STAT_INC(rx_big_nobuf);
return (NULL);
}
static inline void
myri10ge_tx_done(struct myri10ge_slice_state *ss, uint32_t mcp_index)
{
myri10ge_tx_ring_t *tx;
struct myri10ge_tx_dma_handle_head handles;
int idx;
int limit = 0;
tx = &ss->tx;
handles.head = NULL;
handles.tail = NULL;
while (tx->pkt_done != (int)mcp_index) {
idx = tx->done & tx->mask;
if (tx->info[idx].m) {
(void) ddi_dma_unbind_handle(tx->info[idx].handle->h);
tx->info[idx].handle->next = handles.head;
handles.head = tx->info[idx].handle;
if (handles.tail == NULL)
handles.tail = tx->info[idx].handle;
freeb(tx->info[idx].m);
tx->info[idx].m = 0;
tx->info[idx].handle = 0;
}
if (tx->info[idx].ostat.opackets != 0) {
tx->stats.multixmt += tx->info[idx].ostat.multixmt;
tx->stats.brdcstxmt += tx->info[idx].ostat.brdcstxmt;
tx->stats.obytes += tx->info[idx].ostat.obytes;
tx->stats.opackets += tx->info[idx].ostat.opackets;
tx->info[idx].stat.un.all = 0;
tx->pkt_done++;
}
tx->done++;
if ((tx->req - tx->done) < (tx->mask >> 1) &&
tx->stall != tx->sched) {
mutex_enter(&ss->tx.lock);
tx->sched = tx->stall;
mutex_exit(&ss->tx.lock);
mac_tx_ring_update(ss->mgp->mh, tx->rh);
}
if (unlikely(++limit > 2 * tx->mask))
break;
}
if (tx->req == tx->done && tx->stop != NULL) {
mutex_enter(&tx->lock);
if (tx->req == tx->done && tx->active) {
*(int *)(void *)tx->stop = 1;
tx->active = 0;
mb();
}
mutex_exit(&tx->lock);
}
if (handles.head != NULL)
myri10ge_free_tx_handles(tx, &handles);
}
static void
myri10ge_mbl_init(struct myri10ge_mblk_list *mbl)
{
mbl->head = NULL;
mbl->tail = &mbl->head;
mbl->cnt = 0;
}
void
myri10ge_mbl_append(struct myri10ge_slice_state *ss,
struct myri10ge_mblk_list *mbl, mblk_t *mp)
{
*(mbl->tail) = mp;
mbl->tail = &mp->b_next;
mp->b_next = NULL;
mbl->cnt++;
}
static inline void
myri10ge_clean_rx_done(struct myri10ge_slice_state *ss,
struct myri10ge_mblk_list *mbl, int limit, boolean_t *stop)
{
myri10ge_rx_done_t *rx_done = &ss->rx_done;
struct myri10ge_priv *mgp = ss->mgp;
mblk_t *mp;
struct lro_entry *lro;
uint16_t length;
uint16_t checksum;
while (rx_done->entry[rx_done->idx].length != 0) {
if (unlikely (*stop)) {
break;
}
length = ntohs(rx_done->entry[rx_done->idx].length);
length &= (~MXGEFW_RSS_HASH_MASK);
limit -= length;
if (unlikely(limit < 0))
break;
rx_done->entry[rx_done->idx].length = 0;
checksum = ntohs(rx_done->entry[rx_done->idx].checksum);
if (length <= myri10ge_small_bytes)
mp = myri10ge_rx_done_small(ss, length, checksum);
else
mp = myri10ge_rx_done_big(ss, length, checksum);
if (mp != NULL) {
if (!myri10ge_lro ||
0 != myri10ge_lro_rx(ss, mp, checksum, mbl))
myri10ge_mbl_append(ss, mbl, mp);
}
rx_done->cnt++;
rx_done->idx = rx_done->cnt & (mgp->max_intr_slots - 1);
}
while (ss->lro_active != NULL) {
lro = ss->lro_active;
ss->lro_active = lro->next;
myri10ge_lro_flush(ss, lro, mbl);
}
}
static void
myri10ge_intr_rx(struct myri10ge_slice_state *ss)
{
uint64_t gen;
struct myri10ge_mblk_list mbl;
myri10ge_mbl_init(&mbl);
if (mutex_tryenter(&ss->rx_lock) == 0)
return;
gen = ss->rx_gen_num;
myri10ge_clean_rx_done(ss, &mbl, MYRI10GE_POLL_NULL,
&ss->rx_polling);
if (mbl.head != NULL)
mac_rx_ring(ss->mgp->mh, ss->rx_rh, mbl.head, gen);
mutex_exit(&ss->rx_lock);
}
static mblk_t *
myri10ge_poll_rx(void *arg, int bytes)
{
struct myri10ge_slice_state *ss = arg;
struct myri10ge_mblk_list mbl;
boolean_t dummy = B_FALSE;
if (bytes == 0)
return (NULL);
myri10ge_mbl_init(&mbl);
mutex_enter(&ss->rx_lock);
if (ss->rx_polling)
myri10ge_clean_rx_done(ss, &mbl, bytes, &dummy);
else
printf("%d: poll_rx: token=%d, polling=%d\n", (int)(ss -
ss->mgp->ss), ss->rx_token, ss->rx_polling);
mutex_exit(&ss->rx_lock);
return (mbl.head);
}
static uint_t
myri10ge_intr(caddr_t arg0, caddr_t arg1)
{
struct myri10ge_slice_state *ss =
(struct myri10ge_slice_state *)(void *)arg0;
struct myri10ge_priv *mgp = ss->mgp;
mcp_irq_data_t *stats = ss->fw_stats;
myri10ge_tx_ring_t *tx = &ss->tx;
uint32_t send_done_count;
uint8_t valid;
if (!stats->valid) {
return (DDI_INTR_UNCLAIMED);
}
valid = stats->valid;
if (valid & 1)
myri10ge_intr_rx(ss);
if (mgp->ddi_intr_type == DDI_INTR_TYPE_FIXED) {
*mgp->irq_deassert = 0;
if (!myri10ge_deassert_wait)
stats->valid = 0;
mb();
} else {
stats->valid = 0;
}
do {
send_done_count = ntohl(stats->send_done_count);
if (send_done_count != tx->pkt_done)
myri10ge_tx_done(ss, (int)send_done_count);
} while (*((volatile uint8_t *) &stats->valid));
if (stats->stats_updated) {
if (mgp->link_state != stats->link_up || stats->link_down) {
mgp->link_state = stats->link_up;
if (stats->link_down) {
mgp->down_cnt += stats->link_down;
mgp->link_state = 0;
}
if (mgp->link_state) {
if (myri10ge_verbose)
printf("%s: link up\n", mgp->name);
mac_link_update(mgp->mh, LINK_STATE_UP);
} else {
if (myri10ge_verbose)
printf("%s: link down\n", mgp->name);
mac_link_update(mgp->mh, LINK_STATE_DOWN);
}
MYRI10GE_NIC_STAT_INC(link_changes);
}
if (mgp->rdma_tags_available !=
ntohl(ss->fw_stats->rdma_tags_available)) {
mgp->rdma_tags_available =
ntohl(ss->fw_stats->rdma_tags_available);
cmn_err(CE_NOTE, "%s: RDMA timed out! "
"%d tags left\n", mgp->name,
mgp->rdma_tags_available);
}
}
mb();
if (valid & 0x1) {
mutex_enter(&ss->poll_lock);
if (ss->rx_polling) {
ss->rx_token = 1;
} else {
*ss->irq_claim = BE_32(3);
ss->rx_token = 0;
}
mutex_exit(&ss->poll_lock);
}
*(ss->irq_claim + 1) = BE_32(3);
return (DDI_INTR_CLAIMED);
}
static int
myri10ge_m_multicst(void *arg, boolean_t add, const uint8_t *multicastaddr)
{
myri10ge_cmd_t cmd;
struct myri10ge_priv *mgp = arg;
int status, join_leave;
if (add)
join_leave = MXGEFW_JOIN_MULTICAST_GROUP;
else
join_leave = MXGEFW_LEAVE_MULTICAST_GROUP;
(void) memcpy(&cmd.data0, multicastaddr, 4);
(void) memcpy(&cmd.data1, multicastaddr + 4, 2);
cmd.data0 = htonl(cmd.data0);
cmd.data1 = htonl(cmd.data1);
status = myri10ge_send_cmd(mgp, join_leave, &cmd);
if (status == 0)
return (0);
cmn_err(CE_WARN, "%s: failed to set multicast address\n",
mgp->name);
return (status);
}
static int
myri10ge_m_promisc(void *arg, boolean_t on)
{
struct myri10ge_priv *mgp = arg;
myri10ge_change_promisc(mgp, on);
return (0);
}
static inline void
myri10ge_submit_req_backwards(myri10ge_tx_ring_t *tx,
mcp_kreq_ether_send_t *src, int cnt)
{
int idx, starting_slot;
starting_slot = tx->req;
while (cnt > 1) {
cnt--;
idx = (starting_slot + cnt) & tx->mask;
myri10ge_pio_copy(&tx->lanai[idx],
&src[cnt], sizeof (*src));
mb();
}
}
static inline void
myri10ge_submit_req(myri10ge_tx_ring_t *tx, mcp_kreq_ether_send_t *src,
int cnt)
{
int idx, i;
uint32_t *src_ints, *dst_ints;
mcp_kreq_ether_send_t *srcp, *dstp, *dst;
uint8_t last_flags;
idx = tx->req & tx->mask;
last_flags = src->flags;
src->flags = 0;
mb();
dst = dstp = &tx->lanai[idx];
srcp = src;
if ((idx + cnt) < tx->mask) {
for (i = 0; i < (cnt - 1); i += 2) {
myri10ge_pio_copy(dstp, srcp, 2 * sizeof (*src));
mb();
srcp += 2;
dstp += 2;
}
} else {
myri10ge_submit_req_backwards(tx, src, cnt);
i = 0;
}
if (i < cnt) {
myri10ge_pio_copy(dstp, srcp, sizeof (*src));
mb();
}
src->flags |= last_flags;
src_ints = (uint32_t *)src;
src_ints += 3;
dst_ints = (uint32_t *)dst;
dst_ints += 3;
*dst_ints = *src_ints;
tx->req += cnt;
mb();
if (!tx->active && tx->go != NULL) {
*(int *)(void *)tx->go = 1;
tx->active = 1;
tx->activate++;
mb();
}
}
static inline void
myri10ge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags)
{
uint32_t lso_flag;
mac_lso_get(mp, mss, &lso_flag);
(*flags) |= lso_flag;
}
static int
myri10ge_pullup(struct myri10ge_slice_state *ss, mblk_t *mp)
{
uint32_t start, stuff, tx_offload_flags, mss;
int ok;
mss = 0;
mac_hcksum_get(mp, &start, &stuff, NULL, NULL, &tx_offload_flags);
myri10ge_lso_info_get(mp, &mss, &tx_offload_flags);
ok = pullupmsg(mp, -1);
if (!ok) {
printf("pullupmsg failed");
return (DDI_FAILURE);
}
MYRI10GE_ATOMIC_SLICE_STAT_INC(xmit_pullup);
mac_hcksum_set(mp, start, stuff, 0, 0, tx_offload_flags);
if (tx_offload_flags & HW_LSO)
DB_LSOMSS(mp) = (uint16_t)mss;
lso_info_set(mp, mss, tx_offload_flags);
return (DDI_SUCCESS);
}
static inline void
myri10ge_tx_stat(struct myri10ge_tx_pkt_stats *s, struct ether_header *eh,
int opackets, int obytes)
{
s->un.all = 0;
if (eh->ether_dhost.ether_addr_octet[0] & 1) {
if (0 == (bcmp(eh->ether_dhost.ether_addr_octet,
myri10ge_broadcastaddr, sizeof (eh->ether_dhost))))
s->un.s.brdcstxmt = 1;
else
s->un.s.multixmt = 1;
}
s->un.s.opackets = (uint16_t)opackets;
s->un.s.obytes = obytes;
}
static int
myri10ge_tx_copy(struct myri10ge_slice_state *ss, mblk_t *mp,
mcp_kreq_ether_send_t *req)
{
myri10ge_tx_ring_t *tx = &ss->tx;
caddr_t ptr;
struct myri10ge_tx_copybuf *cp;
mblk_t *bp;
int idx, mblen, avail;
uint16_t len;
mutex_enter(&tx->lock);
avail = tx->mask - (tx->req - tx->done);
if (avail <= 1) {
mutex_exit(&tx->lock);
return (EBUSY);
}
idx = tx->req & tx->mask;
cp = &tx->cp[idx];
ptr = cp->va;
for (len = 0, bp = mp; bp != NULL; bp = bp->b_cont) {
mblen = MBLKL(bp);
bcopy(bp->b_rptr, ptr, mblen);
ptr += mblen;
len += mblen;
}
if (len < 60) {
bzero(ptr, 64 - len);
len = 60;
}
req->addr_low = cp->dma.low;
req->addr_high = cp->dma.high;
req->length = htons(len);
req->pad = 0;
req->rdma_count = 1;
myri10ge_tx_stat(&tx->info[idx].stat,
(struct ether_header *)(void *)cp->va, 1, len);
(void) ddi_dma_sync(cp->dma.handle, 0, len, DDI_DMA_SYNC_FORDEV);
myri10ge_submit_req(&ss->tx, req, 1);
mutex_exit(&tx->lock);
freemsg(mp);
return (DDI_SUCCESS);
}
static void
myri10ge_send_locked(myri10ge_tx_ring_t *tx, mcp_kreq_ether_send_t *req_list,
struct myri10ge_tx_buffer_state *tx_info,
int count)
{
int i, idx;
idx = 0;
for (i = 0; i < count; i++) {
idx = (tx->req + i) & tx->mask;
tx->info[idx].m = tx_info[i].m;
tx->info[idx].handle = tx_info[i].handle;
}
tx->info[idx].stat.un.all = tx_info[0].stat.un.all;
myri10ge_submit_req(tx, req_list, count);
}
static void
myri10ge_copydata(mblk_t *mp, int off, int len, caddr_t buf)
{
mblk_t *bp;
int seglen;
uint_t count;
bp = mp;
while (off > 0) {
seglen = MBLKL(bp);
if (off < seglen)
break;
off -= seglen;
bp = bp->b_cont;
}
while (len > 0) {
seglen = MBLKL(bp);
count = min(seglen - off, len);
bcopy(bp->b_rptr + off, buf, count);
len -= count;
buf += count;
off = 0;
bp = bp->b_cont;
}
}
static int
myri10ge_ether_parse_header(mblk_t *mp)
{
struct ether_header eh_copy;
struct ether_header *eh;
int eth_hdr_len, seglen;
seglen = MBLKL(mp);
eth_hdr_len = sizeof (*eh);
if (seglen < eth_hdr_len) {
myri10ge_copydata(mp, 0, eth_hdr_len, (caddr_t)&eh_copy);
eh = &eh_copy;
} else {
eh = (struct ether_header *)(void *)mp->b_rptr;
}
if (eh->ether_type == BE_16(ETHERTYPE_VLAN)) {
eth_hdr_len += 4;
}
return (eth_hdr_len);
}
static int
myri10ge_lso_parse_header(mblk_t *mp, int off)
{
char buf[128];
int seglen, sum_off;
struct ip *ip;
struct tcphdr *tcp;
seglen = MBLKL(mp);
if (seglen < off + sizeof (*ip)) {
myri10ge_copydata(mp, off, sizeof (*ip), buf);
ip = (struct ip *)(void *)buf;
} else {
ip = (struct ip *)(void *)(mp->b_rptr + off);
}
if (seglen < off + (ip->ip_hl << 2) + sizeof (*tcp)) {
myri10ge_copydata(mp, off,
(ip->ip_hl << 2) + sizeof (*tcp), buf);
ip = (struct ip *)(void *)buf;
}
tcp = (struct tcphdr *)(void *)((char *)ip + (ip->ip_hl << 2));
if (ip->ip_sum != 0) {
if (((char *)ip) != buf) {
ip->ip_sum = 0;
} else {
sum_off = off + _PTRDIFF(&ip->ip_sum, buf);
while (sum_off > (int)(MBLKL(mp) - 1)) {
sum_off -= MBLKL(mp);
mp = mp->b_cont;
}
mp->b_rptr[sum_off] = 0;
sum_off++;
while (sum_off > MBLKL(mp) - 1) {
sum_off -= MBLKL(mp);
mp = mp->b_cont;
}
mp->b_rptr[sum_off] = 0;
}
}
return (off + ((ip->ip_hl + tcp->th_off) << 2));
}
static int
myri10ge_tx_tso_copy(struct myri10ge_slice_state *ss, mblk_t *mp,
mcp_kreq_ether_send_t *req_list, int hdr_size, int pkt_size,
uint16_t mss, uint8_t cksum_offset)
{
myri10ge_tx_ring_t *tx = &ss->tx;
struct myri10ge_priv *mgp = ss->mgp;
mblk_t *bp;
mcp_kreq_ether_send_t *req;
struct myri10ge_tx_copybuf *cp;
caddr_t rptr, ptr;
int mblen, count, cum_len, mss_resid, tx_req, pkt_size_tmp;
int resid, avail, idx, hdr_size_tmp, tx_boundary;
int rdma_count;
uint32_t seglen, len, boundary, low, high_swapped;
uint16_t pseudo_hdr_offset = htons(mss);
uint8_t flags;
tx_boundary = mgp->tx_boundary;
hdr_size_tmp = hdr_size;
resid = tx_boundary;
count = 1;
mutex_enter(&tx->lock);
avail = tx->mask - (tx->req - tx->done);
if (unlikely(avail <= MYRI10GE_MAX_SEND_DESC_TSO)) {
atomic_inc_32(&tx->stall);
mutex_exit(&tx->lock);
return (EBUSY);
}
cum_len = -hdr_size;
count = 0;
req = req_list;
idx = tx->mask & tx->req;
cp = &tx->cp[idx];
low = ntohl(cp->dma.low);
ptr = cp->va;
cp->len = 0;
if (mss) {
int payload = pkt_size - hdr_size;
uint16_t opackets = (payload / mss) + ((payload % mss) != 0);
tx->info[idx].ostat.opackets = opackets;
tx->info[idx].ostat.obytes = (opackets - 1) * hdr_size
+ pkt_size;
}
hdr_size_tmp = hdr_size;
mss_resid = mss;
flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST);
tx_req = tx->req;
for (bp = mp; bp != NULL; bp = bp->b_cont) {
mblen = MBLKL(bp);
rptr = (caddr_t)bp->b_rptr;
len = min(hdr_size_tmp, mblen);
if (len) {
bcopy(rptr, ptr, len);
rptr += len;
ptr += len;
resid -= len;
mblen -= len;
hdr_size_tmp -= len;
cp->len += len;
if (hdr_size_tmp)
continue;
if (resid < mss) {
tx_req++;
idx = tx->mask & tx_req;
cp = &tx->cp[idx];
low = ntohl(cp->dma.low);
ptr = cp->va;
resid = tx_boundary;
}
}
while (mblen) {
len = min(mss_resid, mblen);
bcopy(rptr, ptr, len);
mss_resid -= len;
resid -= len;
mblen -= len;
rptr += len;
ptr += len;
cp->len += len;
if (mss_resid == 0) {
mss_resid = mss;
if (resid < mss) {
tx_req++;
idx = tx->mask & tx_req;
cp = &tx->cp[idx];
cp->len = 0;
low = ntohl(cp->dma.low);
ptr = cp->va;
resid = tx_boundary;
}
}
}
}
req = req_list;
pkt_size_tmp = pkt_size;
count = 0;
rdma_count = 0;
tx_req = tx->req;
while (pkt_size_tmp) {
idx = tx->mask & tx_req;
cp = &tx->cp[idx];
high_swapped = cp->dma.high;
low = ntohl(cp->dma.low);
len = cp->len;
if (len == 0) {
printf("len=0! pkt_size_tmp=%d, pkt_size=%d\n",
pkt_size_tmp, pkt_size);
for (bp = mp; bp != NULL; bp = bp->b_cont) {
mblen = MBLKL(bp);
printf("mblen:%d\n", mblen);
}
pkt_size_tmp = pkt_size;
tx_req = tx->req;
while (pkt_size_tmp > 0) {
idx = tx->mask & tx_req;
cp = &tx->cp[idx];
printf("cp->len = %d\n", cp->len);
pkt_size_tmp -= cp->len;
tx_req++;
}
printf("dropped\n");
MYRI10GE_ATOMIC_SLICE_STAT_INC(xmit_err);
goto done;
}
pkt_size_tmp -= len;
while (len) {
while (len) {
uint8_t flags_next;
int cum_len_next;
boundary = (low + mgp->tx_boundary) &
~(mgp->tx_boundary - 1);
seglen = boundary - low;
if (seglen > len)
seglen = len;
flags_next = flags & ~MXGEFW_FLAGS_FIRST;
cum_len_next = cum_len + seglen;
(req-rdma_count)->rdma_count = rdma_count + 1;
if (likely(cum_len >= 0)) {
int next_is_first, chop;
chop = (cum_len_next > mss);
cum_len_next = cum_len_next % mss;
next_is_first = (cum_len_next == 0);
flags |= chop *
MXGEFW_FLAGS_TSO_CHOP;
flags_next |= next_is_first *
MXGEFW_FLAGS_FIRST;
rdma_count |= -(chop | next_is_first);
rdma_count += chop & !next_is_first;
} else if (likely(cum_len_next >= 0)) {
int small;
rdma_count = -1;
cum_len_next = 0;
seglen = -cum_len;
small = (mss <= MXGEFW_SEND_SMALL_SIZE);
flags_next = MXGEFW_FLAGS_TSO_PLD |
MXGEFW_FLAGS_FIRST |
(small * MXGEFW_FLAGS_SMALL);
}
req->addr_high = high_swapped;
req->addr_low = htonl(low);
req->pseudo_hdr_offset = pseudo_hdr_offset;
req->pad = 0;
req->rdma_count = 1;
req->cksum_offset = cksum_offset;
req->length = htons(seglen);
req->flags = flags | ((cum_len & 1) *
MXGEFW_FLAGS_ALIGN_ODD);
if (cksum_offset > seglen)
cksum_offset -= seglen;
else
cksum_offset = 0;
low += seglen;
len -= seglen;
cum_len = cum_len_next;
req++;
req->flags = 0;
flags = flags_next;
count++;
rdma_count++;
}
}
tx_req++;
}
(req-rdma_count)->rdma_count = (uint8_t)rdma_count;
do {
req--;
req->flags |= MXGEFW_FLAGS_TSO_LAST;
} while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP |
MXGEFW_FLAGS_FIRST)));
myri10ge_submit_req(tx, req_list, count);
done:
mutex_exit(&tx->lock);
freemsg(mp);
return (DDI_SUCCESS);
}
static int
myri10ge_send(struct myri10ge_slice_state *ss, mblk_t *mp,
mcp_kreq_ether_send_t *req_list, struct myri10ge_tx_buffer_state *tx_info)
{
struct myri10ge_priv *mgp = ss->mgp;
myri10ge_tx_ring_t *tx = &ss->tx;
mcp_kreq_ether_send_t *req;
struct myri10ge_tx_dma_handle *handles, *dma_handle = NULL;
mblk_t *bp;
ddi_dma_cookie_t cookie;
int err, rv, count, avail, mblen, try_pullup, i, max_segs, maclen,
rdma_count, cum_len, lso_hdr_size;
uint32_t start, stuff, tx_offload_flags;
uint32_t seglen, len, mss, boundary, low, high_swapped;
uint_t ncookies;
uint16_t pseudo_hdr_offset;
uint8_t flags, cksum_offset, odd_flag;
int pkt_size;
int lso_copy = myri10ge_lso_copy;
try_pullup = 1;
again:
mac_hcksum_get(mp, &start, &stuff, NULL, NULL, &tx_offload_flags);
myri10ge_lso_info_get(mp, &mss, &tx_offload_flags);
if (tx_offload_flags & HW_LSO) {
max_segs = MYRI10GE_MAX_SEND_DESC_TSO;
if ((tx_offload_flags & HCK_PARTIALCKSUM) == 0) {
MYRI10GE_ATOMIC_SLICE_STAT_INC(xmit_lsobadflags);
freemsg(mp);
return (DDI_SUCCESS);
}
} else {
max_segs = MXGEFW_MAX_SEND_DESC;
mss = 0;
}
req = req_list;
cksum_offset = 0;
pseudo_hdr_offset = 0;
avail = tx->mask - (tx->req - tx->done);
if (avail < max_segs) {
err = EBUSY;
atomic_inc_32(&tx->stall_early);
goto stall;
}
count = 0;
odd_flag = 0;
pkt_size = 0;
flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST);
for (bp = mp; bp != NULL; bp = bp->b_cont) {
dblk_t *dbp;
mblen = MBLKL(bp);
if (mblen == 0) {
MYRI10GE_ATOMIC_SLICE_STAT_INC(xmit_zero_len);
err = EIO;
goto pullup;
}
if (mss && lso_copy == 1 && ((dbp = bp->b_datap) != NULL)) {
if ((void *)dbp->db_lastfree != myri10ge_db_lastfree) {
lso_copy = 0;
}
}
pkt_size += mblen;
count++;
}
if (count >= max_segs) {
err = myri10ge_pullup(ss, mp);
if (likely(err == DDI_SUCCESS)) {
count = 1;
} else {
if (count < MYRI10GE_MAX_SEND_DESC_TSO) {
max_segs = MYRI10GE_MAX_SEND_DESC_TSO;
} else {
MYRI10GE_ATOMIC_SLICE_STAT_INC(xmit_err);
freemsg(mp);
return (0);
}
}
}
cum_len = 0;
maclen = myri10ge_ether_parse_header(mp);
if (tx_offload_flags & HCK_PARTIALCKSUM) {
cksum_offset = start + maclen;
pseudo_hdr_offset = htons(stuff + maclen);
odd_flag = MXGEFW_FLAGS_ALIGN_ODD;
flags |= MXGEFW_FLAGS_CKSUM;
}
lso_hdr_size = 0;
if (mss) {
flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST);
lso_hdr_size = myri10ge_lso_parse_header(mp, maclen);
cum_len = -lso_hdr_size;
if ((mss < mgp->tx_boundary) && lso_copy) {
err = myri10ge_tx_tso_copy(ss, mp, req_list,
lso_hdr_size, pkt_size, mss, cksum_offset);
return (err);
}
pseudo_hdr_offset = htons(mss);
} else if (pkt_size <= MXGEFW_SEND_SMALL_SIZE) {
flags |= MXGEFW_FLAGS_SMALL;
if (pkt_size < myri10ge_tx_copylen) {
req->cksum_offset = cksum_offset;
req->pseudo_hdr_offset = pseudo_hdr_offset;
req->flags = flags;
err = myri10ge_tx_copy(ss, mp, req);
return (err);
}
cum_len = 0;
}
handles = NULL;
err = myri10ge_alloc_tx_handles(ss, count, &handles);
if (err != DDI_SUCCESS) {
err = DDI_FAILURE;
goto stall;
}
count = 0;
rdma_count = 0;
for (bp = mp; bp != NULL; bp = bp->b_cont) {
mblen = MBLKL(bp);
dma_handle = handles;
handles = handles->next;
rv = ddi_dma_addr_bind_handle(dma_handle->h, NULL,
(caddr_t)bp->b_rptr, mblen,
DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL,
&cookie, &ncookies);
if (unlikely(rv != DDI_DMA_MAPPED)) {
err = EIO;
try_pullup = 0;
dma_handle->next = handles;
handles = dma_handle;
goto abort_with_handles;
}
tx_info[count].m = bp;
tx_info[count].handle = dma_handle;
for (; ; ) {
low = MYRI10GE_LOWPART_TO_U32(cookie.dmac_laddress);
high_swapped =
htonl(MYRI10GE_HIGHPART_TO_U32(
cookie.dmac_laddress));
len = (uint32_t)cookie.dmac_size;
while (len) {
uint8_t flags_next;
int cum_len_next;
boundary = (low + mgp->tx_boundary) &
~(mgp->tx_boundary - 1);
seglen = boundary - low;
if (seglen > len)
seglen = len;
flags_next = flags & ~MXGEFW_FLAGS_FIRST;
cum_len_next = cum_len + seglen;
if (mss) {
(req-rdma_count)->rdma_count =
rdma_count + 1;
if (likely(cum_len >= 0)) {
int next_is_first, chop;
chop = (cum_len_next > mss);
cum_len_next =
cum_len_next % mss;
next_is_first =
(cum_len_next == 0);
flags |= chop *
MXGEFW_FLAGS_TSO_CHOP;
flags_next |= next_is_first *
MXGEFW_FLAGS_FIRST;
rdma_count |=
-(chop | next_is_first);
rdma_count +=
chop & !next_is_first;
} else if (likely(cum_len_next >= 0)) {
int small;
rdma_count = -1;
cum_len_next = 0;
seglen = -cum_len;
small = (mss <=
MXGEFW_SEND_SMALL_SIZE);
flags_next =
MXGEFW_FLAGS_TSO_PLD
| MXGEFW_FLAGS_FIRST
| (small *
MXGEFW_FLAGS_SMALL);
}
}
req->addr_high = high_swapped;
req->addr_low = htonl(low);
req->pseudo_hdr_offset = pseudo_hdr_offset;
req->pad = 0;
req->rdma_count = 1;
req->cksum_offset = cksum_offset;
req->length = htons(seglen);
req->flags = flags | ((cum_len & 1) * odd_flag);
if (cksum_offset > seglen)
cksum_offset -= seglen;
else
cksum_offset = 0;
low += seglen;
len -= seglen;
cum_len = cum_len_next;
count++;
rdma_count++;
if (unlikely(count >= max_segs)) {
MYRI10GE_ATOMIC_SLICE_STAT_INC(
xmit_lowbuf);
err = EBUSY;
if (try_pullup)
try_pullup = 2;
goto abort_with_handles;
}
req++;
req->flags = 0;
flags = flags_next;
tx_info[count].m = 0;
}
ncookies--;
if (ncookies == 0)
break;
ddi_dma_nextcookie(dma_handle->h, &cookie);
}
}
(req-rdma_count)->rdma_count = (uint8_t)rdma_count;
if (mss) {
do {
req--;
req->flags |= MXGEFW_FLAGS_TSO_LAST;
} while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP |
MXGEFW_FLAGS_FIRST)));
}
if (mss) {
uint16_t opackets;
int payload;
payload = pkt_size - lso_hdr_size;
opackets = (payload / mss) + ((payload % mss) != 0);
tx_info[0].stat.un.all = 0;
tx_info[0].ostat.opackets = opackets;
tx_info[0].ostat.obytes = (opackets - 1) * lso_hdr_size
+ pkt_size;
} else {
myri10ge_tx_stat(&tx_info[0].stat,
(struct ether_header *)(void *)mp->b_rptr, 1, pkt_size);
}
mutex_enter(&tx->lock);
avail = tx->mask - (tx->req - tx->done);
if (unlikely(avail <= count)) {
mutex_exit(&tx->lock);
err = 0;
goto late_stall;
}
myri10ge_send_locked(tx, req_list, tx_info, count);
mutex_exit(&tx->lock);
return (DDI_SUCCESS);
late_stall:
try_pullup = 0;
atomic_inc_32(&tx->stall_late);
abort_with_handles:
for (i = 0; i < count; i++) {
bp = tx_info[i].m;
tx_info[i].m = 0;
if (bp) {
dma_handle = tx_info[i].handle;
(void) ddi_dma_unbind_handle(dma_handle->h);
dma_handle->next = handles;
handles = dma_handle;
tx_info[i].handle = NULL;
tx_info[i].m = NULL;
}
}
myri10ge_free_tx_handle_slist(tx, handles);
pullup:
if (try_pullup) {
err = myri10ge_pullup(ss, mp);
if (err != DDI_SUCCESS && try_pullup == 2) {
MYRI10GE_ATOMIC_SLICE_STAT_INC(xmit_err);
freemsg(mp);
return (0);
}
try_pullup = 0;
goto again;
}
stall:
if (err != 0) {
if (err == EBUSY) {
atomic_inc_32(&tx->stall);
} else {
MYRI10GE_ATOMIC_SLICE_STAT_INC(xmit_err);
}
}
return (err);
}
static mblk_t *
myri10ge_send_wrapper(void *arg, mblk_t *mp)
{
struct myri10ge_slice_state *ss = arg;
int err = 0;
mcp_kreq_ether_send_t *req_list;
char req_bytes[sizeof (*req_list) * (MYRI10GE_MAX_SEND_DESC_TSO + 4)
+ 8];
struct myri10ge_tx_buffer_state tx_info[MYRI10GE_MAX_SEND_DESC_TSO + 1];
req_list = (struct mcp_kreq_ether_send *)
(((unsigned long)req_bytes + 7UL) & ~7UL);
err = myri10ge_send(ss, mp, req_list, tx_info);
if (err)
return (mp);
else
return (NULL);
}
static int
myri10ge_addmac(void *arg, const uint8_t *mac_addr)
{
struct myri10ge_priv *mgp = arg;
int err;
if (mac_addr == NULL)
return (EINVAL);
mutex_enter(&mgp->intrlock);
if (mgp->macaddr_cnt) {
mutex_exit(&mgp->intrlock);
return (ENOSPC);
}
err = myri10ge_m_unicst(mgp, mac_addr);
if (!err)
mgp->macaddr_cnt++;
mutex_exit(&mgp->intrlock);
if (err)
return (err);
bcopy(mac_addr, mgp->mac_addr, sizeof (mgp->mac_addr));
return (0);
}
static int
myri10ge_remmac(void *arg, const uint8_t *mac_addr)
{
struct myri10ge_priv *mgp = arg;
mutex_enter(&mgp->intrlock);
mgp->macaddr_cnt--;
mutex_exit(&mgp->intrlock);
return (0);
}
static void
myri10ge_fill_group(void *arg, mac_ring_type_t rtype, const int index,
mac_group_info_t *infop, mac_group_handle_t gh)
{
struct myri10ge_priv *mgp = arg;
if (rtype != MAC_RING_TYPE_RX)
return;
infop->mgi_driver = (mac_group_driver_t)mgp;
infop->mgi_start = NULL;
infop->mgi_stop = NULL;
infop->mgi_addmac = myri10ge_addmac;
infop->mgi_remmac = myri10ge_remmac;
infop->mgi_count = mgp->num_slices;
}
static int
myri10ge_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
{
struct myri10ge_slice_state *ss;
ss = (struct myri10ge_slice_state *)rh;
mutex_enter(&ss->rx_lock);
ss->rx_gen_num = mr_gen_num;
mutex_exit(&ss->rx_lock);
return (0);
}
int
myri10ge_rx_ring_stat(mac_ring_driver_t rh, uint_t stat, uint64_t *val)
{
struct myri10ge_slice_state *ss;
ss = (struct myri10ge_slice_state *)rh;
switch (stat) {
case MAC_STAT_RBYTES:
*val = ss->rx_stats.ibytes;
break;
case MAC_STAT_IPACKETS:
*val = ss->rx_stats.ipackets;
break;
default:
*val = 0;
return (ENOTSUP);
}
return (0);
}
int
myri10ge_tx_ring_stat(mac_ring_driver_t rh, uint_t stat, uint64_t *val)
{
struct myri10ge_slice_state *ss;
ss = (struct myri10ge_slice_state *)rh;
switch (stat) {
case MAC_STAT_OBYTES:
*val = ss->tx.stats.obytes;
break;
case MAC_STAT_OPACKETS:
*val = ss->tx.stats.opackets;
break;
default:
*val = 0;
return (ENOTSUP);
}
return (0);
}
static int
myri10ge_rx_ring_intr_disable(mac_intr_handle_t intrh)
{
struct myri10ge_slice_state *ss;
ss = (struct myri10ge_slice_state *)intrh;
mutex_enter(&ss->poll_lock);
ss->rx_polling = B_TRUE;
mutex_exit(&ss->poll_lock);
return (0);
}
static int
myri10ge_rx_ring_intr_enable(mac_intr_handle_t intrh)
{
struct myri10ge_slice_state *ss;
ss = (struct myri10ge_slice_state *)intrh;
mutex_enter(&ss->poll_lock);
ss->rx_polling = B_FALSE;
if (ss->rx_token) {
*ss->irq_claim = BE_32(3);
ss->rx_token = 0;
}
mutex_exit(&ss->poll_lock);
return (0);
}
static void
myri10ge_fill_ring(void *arg, mac_ring_type_t rtype, const int rg_index,
const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
{
struct myri10ge_priv *mgp = arg;
struct myri10ge_slice_state *ss;
mac_intr_t *mintr = &infop->mri_intr;
ASSERT((unsigned int)ring_index < mgp->num_slices);
ss = &mgp->ss[ring_index];
switch (rtype) {
case MAC_RING_TYPE_RX:
ss->rx_rh = rh;
infop->mri_driver = (mac_ring_driver_t)ss;
infop->mri_start = myri10ge_ring_start;
infop->mri_stop = NULL;
infop->mri_poll = myri10ge_poll_rx;
infop->mri_stat = myri10ge_rx_ring_stat;
mintr->mi_handle = (mac_intr_handle_t)ss;
mintr->mi_enable = myri10ge_rx_ring_intr_enable;
mintr->mi_disable = myri10ge_rx_ring_intr_disable;
break;
case MAC_RING_TYPE_TX:
ss->tx.rh = rh;
infop->mri_driver = (mac_ring_driver_t)ss;
infop->mri_start = NULL;
infop->mri_stop = NULL;
infop->mri_tx = myri10ge_send_wrapper;
infop->mri_stat = myri10ge_tx_ring_stat;
break;
default:
break;
}
}
static void
myri10ge_nic_stat_destroy(struct myri10ge_priv *mgp)
{
if (mgp->ksp_stat == NULL)
return;
kstat_delete(mgp->ksp_stat);
mgp->ksp_stat = NULL;
}
static void
myri10ge_slice_stat_destroy(struct myri10ge_slice_state *ss)
{
if (ss->ksp_stat == NULL)
return;
kstat_delete(ss->ksp_stat);
ss->ksp_stat = NULL;
}
static void
myri10ge_info_destroy(struct myri10ge_priv *mgp)
{
if (mgp->ksp_info == NULL)
return;
kstat_delete(mgp->ksp_info);
mgp->ksp_info = NULL;
}
static int
myri10ge_nic_stat_kstat_update(kstat_t *ksp, int rw)
{
struct myri10ge_nic_stat *ethstat;
struct myri10ge_priv *mgp;
mcp_irq_data_t *fw_stats;
if (rw == KSTAT_WRITE)
return (EACCES);
ethstat = (struct myri10ge_nic_stat *)ksp->ks_data;
mgp = (struct myri10ge_priv *)ksp->ks_private;
fw_stats = mgp->ss[0].fw_stats;
ethstat->dma_read_bw_MBs.value.ul = mgp->read_dma;
ethstat->dma_write_bw_MBs.value.ul = mgp->write_dma;
ethstat->dma_read_write_bw_MBs.value.ul = mgp->read_write_dma;
if (myri10ge_tx_dma_attr.dma_attr_flags & DDI_DMA_FORCE_PHYSICAL)
ethstat->dma_force_physical.value.ul = 1;
else
ethstat->dma_force_physical.value.ul = 0;
ethstat->lanes.value.ul = mgp->pcie_link_width;
ethstat->dropped_bad_crc32.value.ul =
ntohl(fw_stats->dropped_bad_crc32);
ethstat->dropped_bad_phy.value.ul =
ntohl(fw_stats->dropped_bad_phy);
ethstat->dropped_link_error_or_filtered.value.ul =
ntohl(fw_stats->dropped_link_error_or_filtered);
ethstat->dropped_link_overflow.value.ul =
ntohl(fw_stats->dropped_link_overflow);
ethstat->dropped_multicast_filtered.value.ul =
ntohl(fw_stats->dropped_multicast_filtered);
ethstat->dropped_no_big_buffer.value.ul =
ntohl(fw_stats->dropped_no_big_buffer);
ethstat->dropped_no_small_buffer.value.ul =
ntohl(fw_stats->dropped_no_small_buffer);
ethstat->dropped_overrun.value.ul =
ntohl(fw_stats->dropped_overrun);
ethstat->dropped_pause.value.ul =
ntohl(fw_stats->dropped_pause);
ethstat->dropped_runt.value.ul =
ntohl(fw_stats->dropped_runt);
ethstat->link_up.value.ul =
ntohl(fw_stats->link_up);
ethstat->dropped_unicast_filtered.value.ul =
ntohl(fw_stats->dropped_unicast_filtered);
return (0);
}
static int
myri10ge_slice_stat_kstat_update(kstat_t *ksp, int rw)
{
struct myri10ge_slice_stat *ethstat;
struct myri10ge_slice_state *ss;
if (rw == KSTAT_WRITE)
return (EACCES);
ethstat = (struct myri10ge_slice_stat *)ksp->ks_data;
ss = (struct myri10ge_slice_state *)ksp->ks_private;
ethstat->rx_big.value.ul = ss->j_rx_cnt;
ethstat->rx_bigbuf_firmware.value.ul = ss->rx_big.cnt - ss->j_rx_cnt;
ethstat->rx_bigbuf_pool.value.ul =
ss->jpool.num_alloc - ss->jbufs_for_smalls;
ethstat->rx_bigbuf_smalls.value.ul = ss->jbufs_for_smalls;
ethstat->rx_small.value.ul = ss->rx_small.cnt -
(ss->rx_small.mask + 1);
ethstat->tx_done.value.ul = ss->tx.done;
ethstat->tx_req.value.ul = ss->tx.req;
ethstat->tx_activate.value.ul = ss->tx.activate;
ethstat->xmit_sched.value.ul = ss->tx.sched;
ethstat->xmit_stall.value.ul = ss->tx.stall;
ethstat->xmit_stall_early.value.ul = ss->tx.stall_early;
ethstat->xmit_stall_late.value.ul = ss->tx.stall_late;
ethstat->xmit_err.value.ul = MYRI10GE_SLICE_STAT(xmit_err);
return (0);
}
static int
myri10ge_info_kstat_update(kstat_t *ksp, int rw)
{
struct myri10ge_info *info;
struct myri10ge_priv *mgp;
if (rw == KSTAT_WRITE)
return (EACCES);
info = (struct myri10ge_info *)ksp->ks_data;
mgp = (struct myri10ge_priv *)ksp->ks_private;
kstat_named_setstr(&info->driver_version, MYRI10GE_VERSION_STR);
kstat_named_setstr(&info->firmware_version, mgp->fw_version);
kstat_named_setstr(&info->firmware_name, mgp->fw_name);
kstat_named_setstr(&info->interrupt_type, mgp->intr_type);
kstat_named_setstr(&info->product_code, mgp->pc_str);
kstat_named_setstr(&info->serial_number, mgp->sn_str);
return (0);
}
static struct myri10ge_info myri10ge_info_template = {
{ "driver_version", KSTAT_DATA_STRING },
{ "firmware_version", KSTAT_DATA_STRING },
{ "firmware_name", KSTAT_DATA_STRING },
{ "interrupt_type", KSTAT_DATA_STRING },
{ "product_code", KSTAT_DATA_STRING },
{ "serial_number", KSTAT_DATA_STRING },
};
static kmutex_t myri10ge_info_template_lock;
static int
myri10ge_info_init(struct myri10ge_priv *mgp)
{
struct kstat *ksp;
ksp = kstat_create("myri10ge", ddi_get_instance(mgp->dip),
"myri10ge_info", "net", KSTAT_TYPE_NAMED,
sizeof (myri10ge_info_template) /
sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
if (ksp == NULL) {
cmn_err(CE_WARN,
"%s: myri10ge_info_init: kstat_create failed", mgp->name);
return (DDI_FAILURE);
}
mgp->ksp_info = ksp;
ksp->ks_update = myri10ge_info_kstat_update;
ksp->ks_private = (void *) mgp;
ksp->ks_data = &myri10ge_info_template;
ksp->ks_lock = &myri10ge_info_template_lock;
if (MYRI10GE_VERSION_STR != NULL)
ksp->ks_data_size += strlen(MYRI10GE_VERSION_STR) + 1;
ksp->ks_data_size += strlen(mgp->fw_version) + 1;
ksp->ks_data_size += strlen(mgp->fw_name) + 1;
ksp->ks_data_size += strlen(mgp->intr_type) + 1;
if (mgp->pc_str != NULL)
ksp->ks_data_size += strlen(mgp->pc_str) + 1;
if (mgp->sn_str != NULL)
ksp->ks_data_size += strlen(mgp->sn_str) + 1;
kstat_install(ksp);
return (DDI_SUCCESS);
}
static int
myri10ge_nic_stat_init(struct myri10ge_priv *mgp)
{
struct kstat *ksp;
struct myri10ge_nic_stat *ethstat;
ksp = kstat_create("myri10ge", ddi_get_instance(mgp->dip),
"myri10ge_nic_stats", "net", KSTAT_TYPE_NAMED,
sizeof (*ethstat) / sizeof (kstat_named_t), 0);
if (ksp == NULL) {
cmn_err(CE_WARN,
"%s: myri10ge_stat_init: kstat_create failed", mgp->name);
return (DDI_FAILURE);
}
mgp->ksp_stat = ksp;
ethstat = (struct myri10ge_nic_stat *)(ksp->ks_data);
kstat_named_init(ðstat->dma_read_bw_MBs,
"dma_read_bw_MBs", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dma_write_bw_MBs,
"dma_write_bw_MBs", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dma_read_write_bw_MBs,
"dma_read_write_bw_MBs", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dma_force_physical,
"dma_force_physical", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->lanes,
"lanes", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dropped_bad_crc32,
"dropped_bad_crc32", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dropped_bad_phy,
"dropped_bad_phy", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dropped_link_error_or_filtered,
"dropped_link_error_or_filtered", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dropped_link_overflow,
"dropped_link_overflow", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dropped_multicast_filtered,
"dropped_multicast_filtered", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dropped_no_big_buffer,
"dropped_no_big_buffer", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dropped_no_small_buffer,
"dropped_no_small_buffer", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dropped_overrun,
"dropped_overrun", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dropped_pause,
"dropped_pause", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dropped_runt,
"dropped_runt", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dropped_unicast_filtered,
"dropped_unicast_filtered", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->dropped_runt, "dropped_runt",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->link_up, "link_up", KSTAT_DATA_ULONG);
kstat_named_init(ðstat->link_changes, "link_changes",
KSTAT_DATA_ULONG);
ksp->ks_update = myri10ge_nic_stat_kstat_update;
ksp->ks_private = (void *) mgp;
kstat_install(ksp);
return (DDI_SUCCESS);
}
static int
myri10ge_slice_stat_init(struct myri10ge_slice_state *ss)
{
struct myri10ge_priv *mgp = ss->mgp;
struct kstat *ksp;
struct myri10ge_slice_stat *ethstat;
int instance;
instance = (ddi_get_instance(mgp->dip) * 1000) + (int)(ss - mgp->ss);
ksp = kstat_create("myri10ge", instance,
"myri10ge_slice_stats", "net", KSTAT_TYPE_NAMED,
sizeof (*ethstat) / sizeof (kstat_named_t), 0);
if (ksp == NULL) {
cmn_err(CE_WARN,
"%s: myri10ge_stat_init: kstat_create failed", mgp->name);
return (DDI_FAILURE);
}
ss->ksp_stat = ksp;
ethstat = (struct myri10ge_slice_stat *)(ksp->ks_data);
kstat_named_init(ðstat->lro_bad_csum, "lro_bad_csum",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->lro_flushed, "lro_flushed",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->lro_queued, "lro_queued",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->rx_bigbuf_firmware, "rx_bigbuf_firmware",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->rx_bigbuf_pool, "rx_bigbuf_pool",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->rx_bigbuf_smalls, "rx_bigbuf_smalls",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->rx_copy, "rx_copy",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->rx_big_nobuf, "rx_big_nobuf",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->rx_small_nobuf, "rx_small_nobuf",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->xmit_zero_len, "xmit_zero_len",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->xmit_pullup, "xmit_pullup",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->xmit_pullup_first, "xmit_pullup_first",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->xmit_lowbuf, "xmit_lowbuf",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->xmit_lsobadflags, "xmit_lsobadflags",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->xmit_sched, "xmit_sched",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->xmit_stall, "xmit_stall",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->xmit_stall_early, "xmit_stall_early",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->xmit_stall_late, "xmit_stall_late",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->xmit_err, "xmit_err",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->tx_req, "tx_req",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->tx_activate, "tx_activate",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->tx_done, "tx_done",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->tx_handles_alloced, "tx_handles_alloced",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->rx_big, "rx_big",
KSTAT_DATA_ULONG);
kstat_named_init(ðstat->rx_small, "rx_small",
KSTAT_DATA_ULONG);
ksp->ks_update = myri10ge_slice_stat_kstat_update;
ksp->ks_private = (void *) ss;
kstat_install(ksp);
return (DDI_SUCCESS);
}
#if defined __i386 || defined i386 || defined __i386__ || defined __x86_64__
#include <vm/hat.h>
#include <sys/ddi_isa.h>
void *device_arena_alloc(size_t size, int vm_flag);
void device_arena_free(void *vaddr, size_t size);
static void
myri10ge_enable_nvidia_ecrc(struct myri10ge_priv *mgp)
{
dev_info_t *parent_dip;
ddi_acc_handle_t handle;
unsigned long bus_number, dev_number, func_number;
unsigned long cfg_pa, paddr, base, pgoffset;
char *cvaddr, *ptr;
uint32_t *ptr32;
int retval = DDI_FAILURE;
int dontcare;
uint16_t read_vid, read_did, vendor_id, device_id;
if (!myri10ge_nvidia_ecrc_enable)
return;
parent_dip = ddi_get_parent(mgp->dip);
if (parent_dip == NULL) {
cmn_err(CE_WARN, "%s: I'm an orphan?", mgp->name);
return;
}
if (pci_config_setup(parent_dip, &handle) != DDI_SUCCESS) {
cmn_err(CE_WARN,
"%s: Could not access my parent's registers", mgp->name);
return;
}
vendor_id = pci_config_get16(handle, PCI_CONF_VENID);
device_id = pci_config_get16(handle, PCI_CONF_DEVID);
pci_config_teardown(&handle);
if (myri10ge_verbose) {
unsigned long bus_number, dev_number, func_number;
int reg_set, span;
(void) myri10ge_reg_set(parent_dip, ®_set, &span,
&bus_number, &dev_number, &func_number);
if (myri10ge_verbose)
printf("%s: parent at %ld:%ld:%ld\n", mgp->name,
bus_number, dev_number, func_number);
}
if (vendor_id != 0x10de)
return;
if (device_id != 0x005d &&
(device_id < 0x374 || device_id > 0x378) ) {
return;
}
(void) myri10ge_reg_set(parent_dip, &dontcare, &dontcare,
&bus_number, &dev_number, &func_number);
for (cfg_pa = 0xf0000000UL;
retval != DDI_SUCCESS && cfg_pa >= 0xe0000000UL;
cfg_pa -= 0x10000000UL) {
paddr = (cfg_pa + bus_number * 0x00100000UL +
(dev_number * 8 + func_number) * 0x00001000UL);
base = paddr & (~MMU_PAGEOFFSET);
pgoffset = paddr & MMU_PAGEOFFSET;
cvaddr = device_arena_alloc(ptob(1), VM_NOSLEEP);
if (cvaddr == NULL)
cmn_err(CE_WARN, "%s: failed to map nf4: cvaddr\n",
mgp->name);
hat_devload(kas.a_hat, cvaddr, mmu_ptob(1),
i_ddi_paddr_to_pfn(base),
PROT_WRITE|HAT_STRICTORDER, HAT_LOAD_LOCK);
ptr = cvaddr + pgoffset;
read_vid = *(uint16_t *)(void *)(ptr + PCI_CONF_VENID);
read_did = *(uint16_t *)(void *)(ptr + PCI_CONF_DEVID);
if (vendor_id == read_did || device_id == read_did) {
ptr32 = (uint32_t *)(void *)(ptr + 0x178);
if (myri10ge_verbose)
printf("%s: Enabling ECRC on upstream "
"Nvidia bridge (0x%x:0x%x) "
"at %ld:%ld:%ld\n", mgp->name,
read_vid, read_did, bus_number,
dev_number, func_number);
*ptr32 |= 0x40;
retval = DDI_SUCCESS;
}
hat_unload(kas.a_hat, cvaddr, ptob(1), HAT_UNLOAD_UNLOCK);
device_arena_free(cvaddr, ptob(1));
}
}
#else
static void
myri10ge_enable_nvidia_ecrc(struct myri10ge_priv *mgp)
{
}
#endif
static int
myri10ge_firmware_probe(struct myri10ge_priv *mgp)
{
int status;
mgp->tx_boundary = 4096;
if (mgp->max_read_request_4k == 0)
mgp->tx_boundary = 2048;
mgp->fw_name = "rss_eth_z8e";
mgp->eth_z8e = (unsigned char *)rss_eth_z8e;
mgp->eth_z8e_length = rss_eth_z8e_length;
status = myri10ge_load_firmware(mgp);
if (status != 0) {
return (status);
}
myri10ge_enable_nvidia_ecrc(mgp);
status = myri10ge_dma_test(mgp, MXGEFW_CMD_UNALIGNED_TEST);
if (status == 0)
return (0);
if (status != E2BIG)
cmn_err(CE_WARN, "%s: DMA test failed: %d\n",
mgp->name, status);
if (status == ENOSYS)
cmn_err(CE_WARN, "%s: Falling back to ethp! "
"Please install up to date fw\n", mgp->name);
return (status);
}
static int
myri10ge_select_firmware(struct myri10ge_priv *mgp)
{
int aligned;
aligned = 0;
if (myri10ge_force_firmware == 1) {
if (myri10ge_verbose)
printf("%s: Assuming aligned completions (forced)\n",
mgp->name);
aligned = 1;
goto done;
}
if (myri10ge_force_firmware == 2) {
if (myri10ge_verbose)
printf("%s: Assuming unaligned completions (forced)\n",
mgp->name);
aligned = 0;
goto done;
}
if (mgp->pcie_link_width != 0 && mgp->pcie_link_width < 8) {
cmn_err(CE_WARN, "!%s: PCIe link running at x%d\n",
mgp->name, mgp->pcie_link_width);
aligned = 1;
goto done;
}
if (0 == myri10ge_firmware_probe(mgp))
return (0);
done:
if (aligned) {
mgp->fw_name = "rss_eth_z8e";
mgp->eth_z8e = (unsigned char *)rss_eth_z8e;
mgp->eth_z8e_length = rss_eth_z8e_length;
mgp->tx_boundary = 4096;
} else {
mgp->fw_name = "rss_ethp_z8e";
mgp->eth_z8e = (unsigned char *)rss_ethp_z8e;
mgp->eth_z8e_length = rss_ethp_z8e_length;
mgp->tx_boundary = 2048;
}
return (myri10ge_load_firmware(mgp));
}
static int
myri10ge_add_intrs(struct myri10ge_priv *mgp, int add_handler)
{
dev_info_t *devinfo = mgp->dip;
int count, avail, actual, intr_types;
int x, y, rc, inum = 0;
rc = ddi_intr_get_supported_types(devinfo, &intr_types);
if (rc != DDI_SUCCESS) {
cmn_err(CE_WARN,
"!%s: ddi_intr_get_nintrs() failure, rc = %d\n", mgp->name,
rc);
return (DDI_FAILURE);
}
if (!myri10ge_use_msi)
intr_types &= ~DDI_INTR_TYPE_MSI;
if (!myri10ge_use_msix)
intr_types &= ~DDI_INTR_TYPE_MSIX;
if (intr_types & DDI_INTR_TYPE_MSIX) {
mgp->ddi_intr_type = DDI_INTR_TYPE_MSIX;
mgp->intr_type = "MSI-X";
} else if (intr_types & DDI_INTR_TYPE_MSI) {
mgp->ddi_intr_type = DDI_INTR_TYPE_MSI;
mgp->intr_type = "MSI";
} else {
mgp->ddi_intr_type = DDI_INTR_TYPE_FIXED;
mgp->intr_type = "Legacy";
}
rc = ddi_intr_get_nintrs(devinfo, mgp->ddi_intr_type, &count);
if ((rc != DDI_SUCCESS) || (count == 0)) {
cmn_err(CE_WARN, "%s: ddi_intr_get_nintrs() failure, rc: %d, "
"count: %d", mgp->name, rc, count);
return (DDI_FAILURE);
}
rc = ddi_intr_get_navail(devinfo, mgp->ddi_intr_type, &avail);
if ((rc != DDI_SUCCESS) || (avail == 0)) {
cmn_err(CE_WARN, "%s: ddi_intr_get_navail() failure, "
"rc: %d, avail: %d\n", mgp->name, rc, avail);
return (DDI_FAILURE);
}
if (avail < count) {
cmn_err(CE_NOTE,
"!%s: nintrs() returned %d, navail returned %d",
mgp->name, count, avail);
count = avail;
}
if (count < mgp->num_slices)
return (DDI_FAILURE);
if (count > mgp->num_slices)
count = mgp->num_slices;
mgp->intr_size = count * sizeof (ddi_intr_handle_t);
mgp->htable = kmem_alloc(mgp->intr_size, KM_SLEEP);
rc = ddi_intr_alloc(devinfo, mgp->htable, mgp->ddi_intr_type, inum,
count, &actual, DDI_INTR_ALLOC_NORMAL);
if ((rc != DDI_SUCCESS) || (actual == 0)) {
cmn_err(CE_WARN, "%s: ddi_intr_alloc() failed: %d",
mgp->name, rc);
kmem_free(mgp->htable, mgp->intr_size);
mgp->htable = NULL;
return (DDI_FAILURE);
}
if ((actual < count) && myri10ge_verbose) {
cmn_err(CE_NOTE, "%s: got %d/%d slices",
mgp->name, actual, count);
}
mgp->intr_cnt = actual;
if (ddi_intr_get_pri(mgp->htable[0], &mgp->intr_pri)
!= DDI_SUCCESS) {
cmn_err(CE_WARN, "%s: ddi_intr_get_pri() failed", mgp->name);
for (y = 0; y < actual; y++) {
(void) ddi_intr_free(mgp->htable[y]);
}
kmem_free(mgp->htable, mgp->intr_size);
mgp->htable = NULL;
return (DDI_FAILURE);
}
mgp->icookie = (void *)(uintptr_t)mgp->intr_pri;
if (!add_handler)
return (DDI_SUCCESS);
for (x = 0; x < actual; x++) {
if (ddi_intr_add_handler(mgp->htable[x], myri10ge_intr,
(caddr_t)&mgp->ss[x], NULL) != DDI_SUCCESS) {
cmn_err(CE_WARN, "%s: ddi_intr_add_handler() failed",
mgp->name);
for (y = 0; y < actual; y++) {
(void) ddi_intr_free(mgp->htable[y]);
}
kmem_free(mgp->htable, mgp->intr_size);
mgp->htable = NULL;
return (DDI_FAILURE);
}
}
(void) ddi_intr_get_cap(mgp->htable[0], &mgp->intr_cap);
if (mgp->intr_cap & DDI_INTR_FLAG_BLOCK) {
(void) ddi_intr_block_enable(mgp->htable, mgp->intr_cnt);
} else {
for (x = 0; x < mgp->intr_cnt; x++) {
(void) ddi_intr_enable(mgp->htable[x]);
}
}
return (DDI_SUCCESS);
}
static void
myri10ge_rem_intrs(struct myri10ge_priv *mgp, int handler_installed)
{
int x, err;
if (handler_installed) {
if (mgp->intr_cap & DDI_INTR_FLAG_BLOCK) {
(void) ddi_intr_block_disable(mgp->htable,
mgp->intr_cnt);
} else {
for (x = 0; x < mgp->intr_cnt; x++) {
(void) ddi_intr_disable(mgp->htable[x]);
}
}
}
for (x = 0; x < mgp->intr_cnt; x++) {
if (handler_installed) {
err = ddi_intr_remove_handler(mgp->htable[x]);
if (err != DDI_SUCCESS) {
cmn_err(CE_WARN,
"%s: ddi_intr_remove_handler for"
"vec %d returned %d\n", mgp->name,
x, err);
}
}
err = ddi_intr_free(mgp->htable[x]);
if (err != DDI_SUCCESS) {
cmn_err(CE_WARN,
"%s: ddi_intr_free for vec %d returned %d\n",
mgp->name, x, err);
}
}
kmem_free(mgp->htable, mgp->intr_size);
mgp->htable = NULL;
}
static void
myri10ge_test_physical(dev_info_t *dip)
{
ddi_dma_handle_t handle;
struct myri10ge_dma_stuff dma;
void *addr;
int err;
myri10ge_tx_dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
err = ddi_dma_alloc_handle(dip, &myri10ge_tx_dma_attr,
DDI_DMA_DONTWAIT, NULL, &handle);
if (err == DDI_DMA_BADATTR)
goto fail;
ddi_dma_free_handle(&handle);
addr = myri10ge_dma_alloc(dip, 128, &myri10ge_tx_dma_attr,
&myri10ge_dev_access_attr, DDI_DMA_STREAMING,
DDI_DMA_WRITE|DDI_DMA_STREAMING, &dma, 0, DDI_DMA_DONTWAIT);
if (addr == NULL)
goto fail;
myri10ge_dma_free(&dma);
return;
fail:
if (myri10ge_verbose)
printf("myri10ge%d: DDI_DMA_FORCE_PHYSICAL failed, "
"using IOMMU\n", ddi_get_instance(dip));
myri10ge_tx_dma_attr.dma_attr_flags &= ~DDI_DMA_FORCE_PHYSICAL;
}
static void
myri10ge_get_props(dev_info_t *dip)
{
myri10ge_flow_control = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_flow_control", myri10ge_flow_control);
myri10ge_intr_coal_delay = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_intr_coal_delay", myri10ge_intr_coal_delay);
#if defined __i386 || defined i386 || defined __i386__ || defined __x86_64__
myri10ge_nvidia_ecrc_enable = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_nvidia_ecrc_enable", 1);
#endif
myri10ge_use_msi = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_use_msi", myri10ge_use_msi);
myri10ge_deassert_wait = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_deassert_wait", myri10ge_deassert_wait);
myri10ge_verbose = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_verbose", myri10ge_verbose);
myri10ge_tx_copylen = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_tx_copylen", myri10ge_tx_copylen);
if (myri10ge_tx_copylen < 60) {
cmn_err(CE_WARN,
"myri10ge_tx_copylen must be >= 60 bytes\n");
myri10ge_tx_copylen = 60;
}
myri10ge_mtu_override = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_mtu_override", myri10ge_mtu_override);
if (myri10ge_mtu_override >= MYRI10GE_MIN_GLD_MTU &&
myri10ge_mtu_override <= MYRI10GE_MAX_GLD_MTU)
myri10ge_mtu = myri10ge_mtu_override +
sizeof (struct ether_header) + MXGEFW_PAD + VLAN_TAGSZ;
else if (myri10ge_mtu_override != 0) {
cmn_err(CE_WARN,
"myri10ge_mtu_override must be between 1500 and "
"9000 bytes\n");
}
myri10ge_bigbufs_initial = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_bigbufs_initial", myri10ge_bigbufs_initial);
myri10ge_bigbufs_max = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_bigbufs_max", myri10ge_bigbufs_max);
myri10ge_watchdog_reset = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_watchdog_reset", myri10ge_watchdog_reset);
if (myri10ge_bigbufs_initial < 128) {
cmn_err(CE_WARN,
"myri10ge_bigbufs_initial be at least 128\n");
myri10ge_bigbufs_initial = 128;
}
if (myri10ge_bigbufs_max < 128) {
cmn_err(CE_WARN,
"myri10ge_bigbufs_max be at least 128\n");
myri10ge_bigbufs_max = 128;
}
if (myri10ge_bigbufs_max < myri10ge_bigbufs_initial) {
cmn_err(CE_WARN,
"myri10ge_bigbufs_max must be >= "
"myri10ge_bigbufs_initial\n");
myri10ge_bigbufs_max = myri10ge_bigbufs_initial;
}
myri10ge_force_firmware = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_force_firmware", myri10ge_force_firmware);
myri10ge_max_slices = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_max_slices", myri10ge_max_slices);
myri10ge_use_msix = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_use_msix", myri10ge_use_msix);
myri10ge_rss_hash = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_rss_hash", myri10ge_rss_hash);
if (myri10ge_rss_hash > MXGEFW_RSS_HASH_TYPE_MAX ||
myri10ge_rss_hash < MXGEFW_RSS_HASH_TYPE_IPV4) {
cmn_err(CE_WARN, "myri10ge: Illegal rssh hash type %d\n",
myri10ge_rss_hash);
myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT;
}
myri10ge_lro = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_lro", myri10ge_lro);
myri10ge_lro_cnt = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_lro_cnt", myri10ge_lro_cnt);
myri10ge_lro_max_aggr = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_lro_max_aggr", myri10ge_lro_max_aggr);
myri10ge_tx_hash = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_tx_hash", myri10ge_tx_hash);
myri10ge_use_lso = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_use_lso", myri10ge_use_lso);
myri10ge_lso_copy = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_lso_copy", myri10ge_lso_copy);
myri10ge_tx_handles_initial = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_tx_handles_initial", myri10ge_tx_handles_initial);
myri10ge_small_bytes = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
"myri10ge_small_bytes", myri10ge_small_bytes);
if ((myri10ge_small_bytes + MXGEFW_PAD) & (128 -1)) {
cmn_err(CE_WARN, "myri10ge: myri10ge_small_bytes (%d)\n",
myri10ge_small_bytes);
cmn_err(CE_WARN, "must be aligned on 128b bndry -2\n");
myri10ge_small_bytes += 128;
myri10ge_small_bytes &= ~(128 -1);
myri10ge_small_bytes -= MXGEFW_PAD;
cmn_err(CE_WARN, "rounded up to %d\n",
myri10ge_small_bytes);
myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT;
}
}
#ifndef PCI_EXP_LNKSTA
#define PCI_EXP_LNKSTA 18
#endif
static int
myri10ge_find_cap(ddi_acc_handle_t handle, uint8_t *capptr, uint8_t capid)
{
uint16_t status;
uint8_t ptr;
status = pci_config_get16(handle, PCI_CONF_STAT);
if (!(status & PCI_STAT_CAP)) {
cmn_err(CE_WARN, "PCI_STAT_CAP not found\n");
return (ENXIO);
}
ptr = pci_config_get8(handle, PCI_CONF_CAP_PTR);
while (ptr != PCI_CAP_NEXT_PTR_NULL) {
if (pci_config_get8(handle, ptr + PCI_CAP_ID) == capid)
break;
ptr = pci_config_get8(handle, ptr + PCI_CAP_NEXT_PTR);
}
if (ptr < 64) {
cmn_err(CE_WARN, "Bad capability offset %d\n", ptr);
return (ENXIO);
}
*capptr = ptr;
return (0);
}
static int
myri10ge_set_max_readreq(ddi_acc_handle_t handle)
{
int err;
uint16_t val;
uint8_t ptr;
err = myri10ge_find_cap(handle, &ptr, PCI_CAP_ID_PCI_E);
if (err != 0) {
cmn_err(CE_WARN, "could not find PCIe cap\n");
return (ENXIO);
}
val = pci_config_get16(handle, ptr + PCIE_DEVCTL);
val = (val & ~PCIE_DEVCTL_MAX_READ_REQ_MASK) |
PCIE_DEVCTL_MAX_READ_REQ_4096;
pci_config_put16(handle, ptr + PCIE_DEVCTL, val);
val = pci_config_get16(handle, ptr + PCIE_DEVCTL);
if ((val & (PCIE_DEVCTL_MAX_READ_REQ_4096)) !=
PCIE_DEVCTL_MAX_READ_REQ_4096) {
cmn_err(CE_WARN, "could not set max read req (%x)\n", val);
return (EINVAL);
}
return (0);
}
static int
myri10ge_read_pcie_link_width(ddi_acc_handle_t handle, int *link)
{
int err;
uint16_t val;
uint8_t ptr;
err = myri10ge_find_cap(handle, &ptr, PCI_CAP_ID_PCI_E);
if (err != 0) {
cmn_err(CE_WARN, "could not set max read req\n");
return (ENXIO);
}
val = pci_config_get16(handle, ptr + PCIE_LINKSTS);
val &= PCIE_LINKSTS_NEG_WIDTH_MASK;
*link = (val >> 4);
return (0);
}
static int
myri10ge_reset_nic(struct myri10ge_priv *mgp)
{
ddi_acc_handle_t handle = mgp->cfg_hdl;
uint32_t reboot;
uint16_t cmd;
int err;
cmd = pci_config_get16(handle, PCI_CONF_COMM);
if ((cmd & PCI_COMM_ME) == 0) {
pci_config_put8(handle, mgp->vso + 0x10, 0x3);
pci_config_put32(handle, mgp->vso + 0x18, 0xfffffff0);
reboot = pci_config_get16(handle, mgp->vso + 0x14);
cmn_err(CE_WARN, "%s NIC rebooted 0x%x\n", mgp->name, reboot);
return (0);
}
if (!myri10ge_watchdog_reset) {
cmn_err(CE_WARN, "%s: not resetting\n", mgp->name);
return (1);
}
myri10ge_stop_locked(mgp);
err = myri10ge_start_locked(mgp);
if (err == DDI_FAILURE) {
return (0);
}
mac_tx_update(mgp->mh);
return (1);
}
static inline int
myri10ge_ring_stalled(myri10ge_tx_ring_t *tx)
{
if (tx->sched != tx->stall &&
tx->done == tx->watchdog_done &&
tx->watchdog_req != tx->watchdog_done)
return (1);
return (0);
}
static void
myri10ge_watchdog(void *arg)
{
struct myri10ge_priv *mgp;
struct myri10ge_slice_state *ss;
myri10ge_tx_ring_t *tx;
int nic_ok = 1;
int slices_stalled, rx_pause, i;
int add_rx;
mgp = arg;
mutex_enter(&mgp->intrlock);
if (mgp->running != MYRI10GE_ETH_RUNNING) {
cmn_err(CE_WARN,
"%s not running, not rearming watchdog (%d)\n",
mgp->name, mgp->running);
mutex_exit(&mgp->intrlock);
return;
}
rx_pause = ntohl(mgp->ss[0].fw_stats->dropped_pause);
for (slices_stalled = 0, i = 0; i < mgp->num_slices; i++) {
tx = &mgp->ss[i].tx;
slices_stalled = myri10ge_ring_stalled(tx);
if (slices_stalled)
break;
}
if (slices_stalled) {
if (mgp->watchdog_rx_pause == rx_pause) {
cmn_err(CE_WARN,
"%s slice %d stalled:(%d, %d, %d, %d, %d %d %d\n)",
mgp->name, i, tx->sched, tx->stall,
tx->done, tx->watchdog_done, tx->req, tx->pkt_done,
(int)ntohl(mgp->ss[i].fw_stats->send_done_count));
nic_ok = myri10ge_reset_nic(mgp);
} else {
cmn_err(CE_WARN,
"%s Flow controlled, check link partner\n",
mgp->name);
}
}
if (!nic_ok) {
cmn_err(CE_WARN,
"%s Nic dead, not rearming watchdog\n", mgp->name);
mutex_exit(&mgp->intrlock);
return;
}
for (i = 0; i < mgp->num_slices; i++) {
ss = &mgp->ss[i];
tx = &ss->tx;
tx->watchdog_done = tx->done;
tx->watchdog_req = tx->req;
if (ss->watchdog_rx_copy != MYRI10GE_SLICE_STAT(rx_copy)) {
ss->watchdog_rx_copy = MYRI10GE_SLICE_STAT(rx_copy);
add_rx =
min(ss->jpool.num_alloc,
myri10ge_bigbufs_max -
(ss->jpool.num_alloc -
ss->jbufs_for_smalls));
if (add_rx != 0) {
(void) myri10ge_add_jbufs(ss, add_rx, 0);
mutex_enter(&ss->jpool.mtx);
myri10ge_restock_jumbos(ss);
mutex_exit(&ss->jpool.mtx);
}
}
}
mgp->watchdog_rx_pause = rx_pause;
mgp->timer_id = timeout(myri10ge_watchdog, mgp,
mgp->timer_ticks);
mutex_exit(&mgp->intrlock);
}
static int
myri10ge_get_coalesce(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *credp)
{
struct myri10ge_priv *mgp = (struct myri10ge_priv *)(void *)cp;
(void) mi_mpprintf(mp, "%d", mgp->intr_coal_delay);
return (0);
}
static int
myri10ge_set_coalesce(queue_t *q, mblk_t *mp, char *value,
caddr_t cp, cred_t *credp)
{
struct myri10ge_priv *mgp = (struct myri10ge_priv *)(void *)cp;
char *end;
size_t new_value;
new_value = mi_strtol(value, &end, 10);
if (end == value)
return (EINVAL);
mutex_enter(&myri10ge_param_lock);
mgp->intr_coal_delay = (int)new_value;
*mgp->intr_coal_delay_ptr = htonl(mgp->intr_coal_delay);
mutex_exit(&myri10ge_param_lock);
return (0);
}
static int
myri10ge_get_pauseparam(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *credp)
{
struct myri10ge_priv *mgp = (struct myri10ge_priv *)(void *)cp;
(void) mi_mpprintf(mp, "%d", mgp->pause);
return (0);
}
static int
myri10ge_set_pauseparam(queue_t *q, mblk_t *mp, char *value,
caddr_t cp, cred_t *credp)
{
struct myri10ge_priv *mgp = (struct myri10ge_priv *)(void *)cp;
char *end;
size_t new_value;
int err = 0;
new_value = mi_strtol(value, &end, 10);
if (end == value)
return (EINVAL);
if (new_value != 0)
new_value = 1;
mutex_enter(&myri10ge_param_lock);
if (new_value != mgp->pause)
err = myri10ge_change_pause(mgp, new_value);
mutex_exit(&myri10ge_param_lock);
return (err);
}
static int
myri10ge_get_int(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *credp)
{
(void) mi_mpprintf(mp, "%d", *(int *)(void *)cp);
return (0);
}
static int
myri10ge_set_int(queue_t *q, mblk_t *mp, char *value,
caddr_t cp, cred_t *credp)
{
char *end;
size_t new_value;
new_value = mi_strtol(value, &end, 10);
if (end == value)
return (EINVAL);
*(int *)(void *)cp = new_value;
return (0);
}
static void
myri10ge_ndd_init(struct myri10ge_priv *mgp)
{
mgp->nd_head = NULL;
(void) nd_load(&mgp->nd_head, "myri10ge_intr_coal_delay",
myri10ge_get_coalesce, myri10ge_set_coalesce, (caddr_t)mgp);
(void) nd_load(&mgp->nd_head, "myri10ge_flow_control",
myri10ge_get_pauseparam, myri10ge_set_pauseparam, (caddr_t)mgp);
(void) nd_load(&mgp->nd_head, "myri10ge_verbose",
myri10ge_get_int, myri10ge_set_int, (caddr_t)&myri10ge_verbose);
(void) nd_load(&mgp->nd_head, "myri10ge_deassert_wait",
myri10ge_get_int, myri10ge_set_int,
(caddr_t)&myri10ge_deassert_wait);
(void) nd_load(&mgp->nd_head, "myri10ge_bigbufs_max",
myri10ge_get_int, myri10ge_set_int,
(caddr_t)&myri10ge_bigbufs_max);
(void) nd_load(&mgp->nd_head, "myri10ge_lro",
myri10ge_get_int, myri10ge_set_int,
(caddr_t)&myri10ge_lro);
(void) nd_load(&mgp->nd_head, "myri10ge_lro_max_aggr",
myri10ge_get_int, myri10ge_set_int,
(caddr_t)&myri10ge_lro_max_aggr);
(void) nd_load(&mgp->nd_head, "myri10ge_tx_hash",
myri10ge_get_int, myri10ge_set_int,
(caddr_t)&myri10ge_tx_hash);
(void) nd_load(&mgp->nd_head, "myri10ge_lso_copy",
myri10ge_get_int, myri10ge_set_int,
(caddr_t)&myri10ge_lso_copy);
}
static void
myri10ge_ndd_fini(struct myri10ge_priv *mgp)
{
nd_free(&mgp->nd_head);
}
static void
myri10ge_m_ioctl(void *arg, queue_t *wq, mblk_t *mp)
{
struct iocblk *iocp;
struct myri10ge_priv *mgp = arg;
int cmd, ok, err;
iocp = (struct iocblk *)(void *)mp->b_rptr;
cmd = iocp->ioc_cmd;
ok = 0;
err = 0;
switch (cmd) {
case ND_GET:
case ND_SET:
ok = nd_getset(wq, mgp->nd_head, mp);
break;
default:
break;
}
if (!ok)
err = EINVAL;
else
err = iocp->ioc_error;
if (!err)
miocack(wq, mp, iocp->ioc_count, err);
else
miocnak(wq, mp, 0, err);
}
static struct myri10ge_priv *mgp_list;
struct myri10ge_priv *
myri10ge_get_instance(uint_t unit)
{
struct myri10ge_priv *mgp;
mutex_enter(&myri10ge_param_lock);
for (mgp = mgp_list; mgp != NULL; mgp = mgp->next) {
if (unit == ddi_get_instance(mgp->dip)) {
mgp->refcnt++;
break;
}
}
mutex_exit(&myri10ge_param_lock);
return (mgp);
}
void
myri10ge_put_instance(struct myri10ge_priv *mgp)
{
mutex_enter(&myri10ge_param_lock);
mgp->refcnt--;
mutex_exit(&myri10ge_param_lock);
}
static boolean_t
myri10ge_m_getcapab(void *arg, mac_capab_t cap, void *cap_data)
{
struct myri10ge_priv *mgp = arg;
uint32_t *cap_hcksum;
mac_capab_lso_t *cap_lso;
mac_capab_rings_t *cap_rings;
switch (cap) {
case MAC_CAPAB_HCKSUM:
cap_hcksum = cap_data;
*cap_hcksum = HCKSUM_INET_PARTIAL;
break;
case MAC_CAPAB_RINGS:
cap_rings = cap_data;
switch (cap_rings->mr_type) {
case MAC_RING_TYPE_RX:
cap_rings->mr_group_type = MAC_GROUP_TYPE_STATIC;
cap_rings->mr_rnum = mgp->num_slices;
cap_rings->mr_gnum = 1;
cap_rings->mr_rget = myri10ge_fill_ring;
cap_rings->mr_gget = myri10ge_fill_group;
break;
case MAC_RING_TYPE_TX:
cap_rings->mr_group_type = MAC_GROUP_TYPE_STATIC;
cap_rings->mr_rnum = mgp->num_slices;
cap_rings->mr_gnum = 0;
cap_rings->mr_rget = myri10ge_fill_ring;
cap_rings->mr_gget = NULL;
break;
default:
return (B_FALSE);
}
break;
case MAC_CAPAB_LSO:
cap_lso = cap_data;
if (!myri10ge_use_lso)
return (B_FALSE);
if (!(mgp->features & MYRI10GE_TSO))
return (B_FALSE);
cap_lso->lso_flags = LSO_TX_BASIC_TCP_IPV4;
cap_lso->lso_basic_tcp_ipv4.lso_max = (uint16_t)-1;
break;
default:
return (B_FALSE);
}
return (B_TRUE);
}
static int
myri10ge_m_stat(void *arg, uint_t stat, uint64_t *val)
{
struct myri10ge_priv *mgp = arg;
struct myri10ge_rx_ring_stats *rstat;
struct myri10ge_tx_ring_stats *tstat;
mcp_irq_data_t *fw_stats = mgp->ss[0].fw_stats;
struct myri10ge_slice_state *ss;
uint64_t tmp = 0;
int i;
switch (stat) {
case MAC_STAT_IFSPEED:
*val = 10ull * 1000ull * 1000000ull;
break;
case MAC_STAT_MULTIRCV:
for (i = 0; i < mgp->num_slices; i++) {
rstat = &mgp->ss[i].rx_stats;
tmp += rstat->multircv;
}
*val = tmp;
break;
case MAC_STAT_BRDCSTRCV:
for (i = 0; i < mgp->num_slices; i++) {
rstat = &mgp->ss[i].rx_stats;
tmp += rstat->brdcstrcv;
}
*val = tmp;
break;
case MAC_STAT_MULTIXMT:
for (i = 0; i < mgp->num_slices; i++) {
tstat = &mgp->ss[i].tx.stats;
tmp += tstat->multixmt;
}
*val = tmp;
break;
case MAC_STAT_BRDCSTXMT:
for (i = 0; i < mgp->num_slices; i++) {
tstat = &mgp->ss[i].tx.stats;
tmp += tstat->brdcstxmt;
}
*val = tmp;
break;
case MAC_STAT_NORCVBUF:
tmp = ntohl(fw_stats->dropped_no_big_buffer);
tmp += ntohl(fw_stats->dropped_no_small_buffer);
tmp += ntohl(fw_stats->dropped_link_overflow);
for (i = 0; i < mgp->num_slices; i++) {
ss = &mgp->ss[i];
tmp += MYRI10GE_SLICE_STAT(rx_big_nobuf);
tmp += MYRI10GE_SLICE_STAT(rx_small_nobuf);
}
*val = tmp;
break;
case MAC_STAT_IERRORS:
tmp += ntohl(fw_stats->dropped_bad_crc32);
tmp += ntohl(fw_stats->dropped_bad_phy);
tmp += ntohl(fw_stats->dropped_runt);
tmp += ntohl(fw_stats->dropped_overrun);
*val = tmp;
break;
case MAC_STAT_OERRORS:
for (i = 0; i < mgp->num_slices; i++) {
ss = &mgp->ss[i];
tmp += MYRI10GE_SLICE_STAT(xmit_lsobadflags);
tmp += MYRI10GE_SLICE_STAT(xmit_err);
}
*val = tmp;
break;
case MAC_STAT_RBYTES:
for (i = 0; i < mgp->num_slices; i++) {
rstat = &mgp->ss[i].rx_stats;
tmp += rstat->ibytes;
}
*val = tmp;
break;
case MAC_STAT_IPACKETS:
for (i = 0; i < mgp->num_slices; i++) {
rstat = &mgp->ss[i].rx_stats;
tmp += rstat->ipackets;
}
*val = tmp;
break;
case MAC_STAT_OBYTES:
for (i = 0; i < mgp->num_slices; i++) {
tstat = &mgp->ss[i].tx.stats;
tmp += tstat->obytes;
}
*val = tmp;
break;
case MAC_STAT_OPACKETS:
for (i = 0; i < mgp->num_slices; i++) {
tstat = &mgp->ss[i].tx.stats;
tmp += tstat->opackets;
}
*val = tmp;
break;
case ETHER_STAT_TOOLONG_ERRORS:
*val = ntohl(fw_stats->dropped_overrun);
break;
#ifdef SOLARIS_S11
case ETHER_STAT_TOOSHORT_ERRORS:
*val = ntohl(fw_stats->dropped_runt);
break;
#endif
case ETHER_STAT_LINK_PAUSE:
*val = mgp->pause;
break;
case ETHER_STAT_LINK_AUTONEG:
*val = 1;
break;
case ETHER_STAT_LINK_DUPLEX:
*val = LINK_DUPLEX_FULL;
break;
default:
return (ENOTSUP);
}
return (0);
}
static void
myri10ge_m_propinfo(void *arg, const char *pr_name,
mac_prop_id_t pr_num, mac_prop_info_handle_t prh)
{
switch (pr_num) {
case MAC_PROP_MTU:
mac_prop_info_set_default_uint32(prh, MYRI10GE_DEFAULT_GLD_MTU);
mac_prop_info_set_range_uint32(prh, MYRI10GE_MIN_GLD_MTU,
MYRI10GE_MAX_GLD_MTU);
break;
default:
break;
}
}
static int
myri10ge_m_setprop(void *arg, const char *pr_name, mac_prop_id_t pr_num,
uint_t pr_valsize, const void *pr_val)
{
int err = 0;
struct myri10ge_priv *mgp = arg;
switch (pr_num) {
case MAC_PROP_MTU: {
uint32_t mtu;
if (pr_valsize < sizeof (mtu)) {
err = EINVAL;
break;
}
bcopy(pr_val, &mtu, sizeof (mtu));
if (mtu > MYRI10GE_MAX_GLD_MTU ||
mtu < MYRI10GE_MIN_GLD_MTU) {
err = EINVAL;
break;
}
mutex_enter(&mgp->intrlock);
if (mgp->running != MYRI10GE_ETH_STOPPED) {
err = EBUSY;
mutex_exit(&mgp->intrlock);
break;
}
myri10ge_mtu = mtu + sizeof (struct ether_header) +
MXGEFW_PAD + VLAN_TAGSZ;
mutex_exit(&mgp->intrlock);
break;
}
default:
err = ENOTSUP;
break;
}
return (err);
}
static mac_callbacks_t myri10ge_m_callbacks = {
(MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_PROPINFO),
myri10ge_m_stat,
myri10ge_m_start,
myri10ge_m_stop,
myri10ge_m_promisc,
myri10ge_m_multicst,
NULL,
NULL,
NULL,
myri10ge_m_ioctl,
myri10ge_m_getcapab,
NULL,
NULL,
myri10ge_m_setprop,
NULL,
myri10ge_m_propinfo
};
static int
myri10ge_probe_slices(struct myri10ge_priv *mgp)
{
myri10ge_cmd_t cmd;
int status;
mgp->num_slices = 1;
(void) memset(&cmd, 0, sizeof (cmd));
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd);
if (status != 0) {
cmn_err(CE_WARN, "%s: failed reset\n", mgp->name);
return (ENXIO);
}
if (myri10ge_use_msix == 0)
return (0);
cmd.data0 = mgp->max_intr_slots * sizeof (struct mcp_slot);
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
if (status != 0) {
cmn_err(CE_WARN, "%s: failed MXGEFW_CMD_SET_INTRQ_SIZE\n",
mgp->name);
return (ENXIO);
}
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES,
&cmd);
if (status != 0)
return (0);
mgp->num_slices = cmd.data0;
if (myri10ge_max_slices == -1)
myri10ge_max_slices = ncpus;
if (mgp->num_slices > myri10ge_max_slices)
mgp->num_slices = myri10ge_max_slices;
while (mgp->num_slices > 1) {
while (!ISP2(mgp->num_slices))
mgp->num_slices--;
if (mgp->num_slices == 1)
return (0);
status = myri10ge_add_intrs(mgp, 0);
if (status == 0) {
myri10ge_rem_intrs(mgp, 0);
if (mgp->intr_cnt == mgp->num_slices) {
if (myri10ge_verbose)
printf("Got %d slices!\n",
mgp->num_slices);
return (0);
}
mgp->num_slices = mgp->intr_cnt;
} else {
mgp->num_slices = mgp->num_slices / 2;
}
}
if (myri10ge_verbose)
printf("Got %d slices\n", mgp->num_slices);
return (0);
}
static void
myri10ge_lro_free(struct myri10ge_slice_state *ss)
{
struct lro_entry *lro;
while (ss->lro_free != NULL) {
lro = ss->lro_free;
ss->lro_free = lro->next;
kmem_free(lro, sizeof (*lro));
}
}
static void
myri10ge_lro_alloc(struct myri10ge_slice_state *ss)
{
struct lro_entry *lro;
int idx;
ss->lro_free = NULL;
ss->lro_active = NULL;
for (idx = 0; idx < myri10ge_lro_cnt; idx++) {
lro = kmem_zalloc(sizeof (*lro), KM_SLEEP);
if (lro == NULL)
continue;
lro->next = ss->lro_free;
ss->lro_free = lro;
}
}
static void
myri10ge_free_slices(struct myri10ge_priv *mgp)
{
struct myri10ge_slice_state *ss;
size_t bytes;
int i;
if (mgp->ss == NULL)
return;
for (i = 0; i < mgp->num_slices; i++) {
ss = &mgp->ss[i];
if (ss->rx_done.entry == NULL)
continue;
myri10ge_dma_free(&ss->rx_done.dma);
ss->rx_done.entry = NULL;
if (ss->fw_stats == NULL)
continue;
myri10ge_dma_free(&ss->fw_stats_dma);
ss->fw_stats = NULL;
mutex_destroy(&ss->rx_lock);
mutex_destroy(&ss->tx.lock);
mutex_destroy(&ss->tx.handle_lock);
mutex_destroy(&ss->poll_lock);
myri10ge_jpool_fini(ss);
myri10ge_slice_stat_destroy(ss);
myri10ge_lro_free(ss);
}
bytes = sizeof (*mgp->ss) * mgp->num_slices;
kmem_free(mgp->ss, bytes);
mgp->ss = NULL;
}
static int
myri10ge_alloc_slices(struct myri10ge_priv *mgp)
{
struct myri10ge_slice_state *ss;
size_t bytes;
int i;
bytes = sizeof (*mgp->ss) * mgp->num_slices;
mgp->ss = kmem_zalloc(bytes, KM_SLEEP);
if (mgp->ss == NULL)
return (ENOMEM);
for (i = 0; i < mgp->num_slices; i++) {
ss = &mgp->ss[i];
ss->mgp = mgp;
bytes = sizeof (*ss->fw_stats);
ss->fw_stats = (mcp_irq_data_t *)(void *)
myri10ge_dma_alloc(mgp->dip, bytes,
&myri10ge_misc_dma_attr, &myri10ge_dev_access_attr,
DDI_DMA_CONSISTENT, DDI_DMA_READ|DDI_DMA_CONSISTENT,
&ss->fw_stats_dma, 1, DDI_DMA_DONTWAIT);
if (ss->fw_stats == NULL)
goto abort;
(void) memset(ss->fw_stats, 0, bytes);
bytes = mgp->max_intr_slots *
sizeof (*ss->rx_done.entry);
ss->rx_done.entry = (mcp_slot_t *)(void *)
myri10ge_dma_alloc(mgp->dip, bytes,
&myri10ge_misc_dma_attr, &myri10ge_dev_access_attr,
DDI_DMA_CONSISTENT, DDI_DMA_READ|DDI_DMA_CONSISTENT,
&ss->rx_done.dma, 1, DDI_DMA_DONTWAIT);
if (ss->rx_done.entry == NULL) {
goto abort;
}
(void) memset(ss->rx_done.entry, 0, bytes);
mutex_init(&ss->rx_lock, NULL, MUTEX_DEFAULT, mgp->icookie);
mutex_init(&ss->tx.lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&ss->tx.handle_lock, NULL, MUTEX_DEFAULT, NULL);
mutex_init(&ss->poll_lock, NULL, MUTEX_DEFAULT, NULL);
myri10ge_jpool_init(ss);
(void) myri10ge_slice_stat_init(ss);
myri10ge_lro_alloc(ss);
}
return (0);
abort:
myri10ge_free_slices(mgp);
return (ENOMEM);
}
static int
myri10ge_save_msi_state(struct myri10ge_priv *mgp,
ddi_acc_handle_t handle)
{
uint8_t ptr;
int err;
err = myri10ge_find_cap(handle, &ptr, PCI_CAP_ID_MSI);
if (err != 0) {
cmn_err(CE_WARN, "%s: could not find MSI cap\n",
mgp->name);
return (DDI_FAILURE);
}
mgp->pci_saved_state.msi_ctrl =
pci_config_get16(handle, ptr + PCI_MSI_CTRL);
mgp->pci_saved_state.msi_addr_low =
pci_config_get32(handle, ptr + PCI_MSI_ADDR_OFFSET);
mgp->pci_saved_state.msi_addr_high =
pci_config_get32(handle, ptr + PCI_MSI_ADDR_OFFSET + 4);
mgp->pci_saved_state.msi_data_32 =
pci_config_get16(handle, ptr + PCI_MSI_32BIT_DATA);
mgp->pci_saved_state.msi_data_64 =
pci_config_get16(handle, ptr + PCI_MSI_64BIT_DATA);
return (DDI_SUCCESS);
}
static int
myri10ge_restore_msi_state(struct myri10ge_priv *mgp,
ddi_acc_handle_t handle)
{
uint8_t ptr;
int err;
err = myri10ge_find_cap(handle, &ptr, PCI_CAP_ID_MSI);
if (err != 0) {
cmn_err(CE_WARN, "%s: could not find MSI cap\n",
mgp->name);
return (DDI_FAILURE);
}
pci_config_put16(handle, ptr + PCI_MSI_CTRL,
mgp->pci_saved_state.msi_ctrl);
pci_config_put32(handle, ptr + PCI_MSI_ADDR_OFFSET,
mgp->pci_saved_state.msi_addr_low);
pci_config_put32(handle, ptr + PCI_MSI_ADDR_OFFSET + 4,
mgp->pci_saved_state.msi_addr_high);
pci_config_put16(handle, ptr + PCI_MSI_32BIT_DATA,
mgp->pci_saved_state.msi_data_32);
pci_config_put16(handle, ptr + PCI_MSI_64BIT_DATA,
mgp->pci_saved_state.msi_data_64);
return (DDI_SUCCESS);
}
static int
myri10ge_save_pci_state(struct myri10ge_priv *mgp)
{
ddi_acc_handle_t handle = mgp->cfg_hdl;
int i;
int err = DDI_SUCCESS;
for (i = 0; i < 16; i++)
mgp->pci_saved_state.base[i] =
pci_config_get32(handle, i*4);
if (mgp->ddi_intr_type == DDI_INTR_TYPE_MSI)
err = myri10ge_save_msi_state(mgp, handle);
return (err);
}
static int
myri10ge_restore_pci_state(struct myri10ge_priv *mgp)
{
ddi_acc_handle_t handle = mgp->cfg_hdl;
int i;
int err = DDI_SUCCESS;
for (i = 15; i >= 0; i--)
pci_config_put32(handle, i*4, mgp->pci_saved_state.base[i]);
if (mgp->ddi_intr_type == DDI_INTR_TYPE_MSI)
err = myri10ge_restore_msi_state(mgp, handle);
if (mgp->max_read_request_4k)
(void) myri10ge_set_max_readreq(handle);
return (err);
}
static int
myri10ge_suspend(dev_info_t *dip)
{
struct myri10ge_priv *mgp = ddi_get_driver_private(dip);
int status;
if (mgp == NULL) {
cmn_err(CE_WARN, "null dip in myri10ge_suspend\n");
return (DDI_FAILURE);
}
if (mgp->dip != dip) {
cmn_err(CE_WARN, "bad dip in myri10ge_suspend\n");
return (DDI_FAILURE);
}
mutex_enter(&mgp->intrlock);
if (mgp->running == MYRI10GE_ETH_RUNNING) {
mgp->running = MYRI10GE_ETH_STOPPING;
mutex_exit(&mgp->intrlock);
(void) untimeout(mgp->timer_id);
mutex_enter(&mgp->intrlock);
myri10ge_stop_locked(mgp);
mgp->running = MYRI10GE_ETH_SUSPENDED_RUNNING;
}
status = myri10ge_save_pci_state(mgp);
mutex_exit(&mgp->intrlock);
return (status);
}
static int
myri10ge_resume(dev_info_t *dip)
{
struct myri10ge_priv *mgp = ddi_get_driver_private(dip);
int status = DDI_SUCCESS;
if (mgp == NULL) {
cmn_err(CE_WARN, "null dip in myri10ge_resume\n");
return (DDI_FAILURE);
}
if (mgp->dip != dip) {
cmn_err(CE_WARN, "bad dip in myri10ge_resume\n");
return (DDI_FAILURE);
}
mutex_enter(&mgp->intrlock);
status = myri10ge_restore_pci_state(mgp);
if (status == DDI_SUCCESS &&
mgp->running == MYRI10GE_ETH_SUSPENDED_RUNNING) {
status = myri10ge_start_locked(mgp);
}
mutex_exit(&mgp->intrlock);
if (status != DDI_SUCCESS)
return (status);
mgp->timer_id = timeout(myri10ge_watchdog, mgp,
mgp->timer_ticks);
return (DDI_SUCCESS);
}
static int
myri10ge_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
struct myri10ge_priv *mgp;
mac_register_t *macp, *omacp;
ddi_acc_handle_t handle;
uint32_t csr, hdr_offset;
int status, span, link_width, max_read_request_4k;
unsigned long bus_number, dev_number, func_number;
size_t bytes;
offset_t ss_offset;
uint8_t vso;
if (cmd == DDI_RESUME) {
return (myri10ge_resume(dip));
}
if (cmd != DDI_ATTACH)
return (DDI_FAILURE);
if (pci_config_setup(dip, &handle) != DDI_SUCCESS)
return (DDI_FAILURE);
csr = pci_config_get32(handle, PCI_CONF_COMM);
pci_config_put32(handle, PCI_CONF_COMM,
(csr |PCI_COMM_ME|PCI_COMM_MAE));
status = myri10ge_read_pcie_link_width(handle, &link_width);
if (status != 0) {
cmn_err(CE_WARN, "could not read link width!\n");
link_width = 0;
}
max_read_request_4k = !myri10ge_set_max_readreq(handle);
status = myri10ge_find_cap(handle, &vso, PCI_CAP_ID_VS);
if (status != 0)
goto abort_with_cfg_hdl;
if ((omacp = mac_alloc(MAC_VERSION)) == NULL)
goto abort_with_cfg_hdl;
macp = kmem_zalloc(sizeof (*macp) * 8, KM_SLEEP);
macp->m_version = omacp->m_version;
if ((mgp = (struct myri10ge_priv *)
kmem_zalloc(sizeof (*mgp), KM_SLEEP)) == NULL) {
goto abort_with_macinfo;
}
ddi_set_driver_private(dip, mgp);
(void) sprintf(mgp->name, "myri10ge%d", ddi_get_instance(dip));
mutex_enter(&myri10ge_param_lock);
myri10ge_get_props(dip);
mgp->intr_coal_delay = myri10ge_intr_coal_delay;
mgp->pause = myri10ge_flow_control;
mutex_exit(&myri10ge_param_lock);
mgp->max_read_request_4k = max_read_request_4k;
mgp->pcie_link_width = link_width;
mgp->running = MYRI10GE_ETH_STOPPED;
mgp->vso = vso;
mgp->dip = dip;
mgp->cfg_hdl = handle;
mgp->timer_ticks = 5 * drv_usectohz(1000000);
myri10ge_test_physical(dip);
bytes = sizeof (*mgp->cmd);
mgp->cmd = (mcp_cmd_response_t *)
(void *)myri10ge_dma_alloc(dip, bytes,
&myri10ge_misc_dma_attr, &myri10ge_dev_access_attr,
DDI_DMA_CONSISTENT, DDI_DMA_RDWR|DDI_DMA_CONSISTENT,
&mgp->cmd_dma, 1, DDI_DMA_DONTWAIT);
if (mgp->cmd == NULL)
goto abort_with_mgp;
(void) myri10ge_reg_set(dip, &mgp->reg_set, &span, &bus_number,
&dev_number, &func_number);
if (myri10ge_verbose)
printf("%s at %ld:%ld:%ld attaching\n", mgp->name,
bus_number, dev_number, func_number);
status = ddi_regs_map_setup(dip, mgp->reg_set, (caddr_t *)&mgp->sram,
(offset_t)0, (offset_t)span, &myri10ge_dev_access_attr,
&mgp->io_handle);
if (status != DDI_SUCCESS) {
cmn_err(CE_WARN, "%s: couldn't map memory space", mgp->name);
printf("%s: reg_set = %d, span = %d, status = %d",
mgp->name, mgp->reg_set, span, status);
goto abort_with_mgp;
}
hdr_offset = *(uint32_t *)(void*)(mgp->sram + MCP_HEADER_PTR_OFFSET);
hdr_offset = ntohl(hdr_offset) & 0xffffc;
ss_offset = hdr_offset +
offsetof(struct mcp_gen_header, string_specs);
mgp->sram_size = ntohl(*(uint32_t *)(void*)(mgp->sram + ss_offset));
myri10ge_pio_copy32(mgp->eeprom_strings,
(uint32_t *)(void*)((char *)mgp->sram + mgp->sram_size),
MYRI10GE_EEPROM_STRINGS_SIZE);
(void) memset(mgp->eeprom_strings +
MYRI10GE_EEPROM_STRINGS_SIZE - 2, 0, 2);
status = myri10ge_read_mac_addr(mgp);
if (status) {
goto abort_with_mapped;
}
status = myri10ge_select_firmware(mgp);
if (status != 0) {
cmn_err(CE_WARN, "%s: failed to load firmware\n", mgp->name);
goto abort_with_mapped;
}
status = myri10ge_probe_slices(mgp);
if (status != 0) {
cmn_err(CE_WARN, "%s: failed to probe slices\n", mgp->name);
goto abort_with_dummy_rdma;
}
status = myri10ge_alloc_slices(mgp);
if (status != 0) {
cmn_err(CE_WARN, "%s: failed to alloc slices\n", mgp->name);
goto abort_with_dummy_rdma;
}
status = myri10ge_add_intrs(mgp, 1);
if (status != 0) {
cmn_err(CE_WARN, "%s: Failed to add interrupt\n",
mgp->name);
goto abort_with_slices;
}
mutex_init(&mgp->cmd_lock, NULL, MUTEX_DRIVER, mgp->icookie);
mutex_init(&mgp->intrlock, NULL, MUTEX_DRIVER, mgp->icookie);
status = myri10ge_nic_stat_init(mgp);
if (status != DDI_SUCCESS)
goto abort_with_interrupts;
status = myri10ge_info_init(mgp);
if (status != DDI_SUCCESS)
goto abort_with_stats;
macp->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
macp->m_driver = mgp;
macp->m_dip = dip;
macp->m_src_addr = mgp->mac_addr;
macp->m_callbacks = &myri10ge_m_callbacks;
macp->m_min_sdu = 0;
macp->m_max_sdu = myri10ge_mtu -
(sizeof (struct ether_header) + MXGEFW_PAD + VLAN_TAGSZ);
#ifdef SOLARIS_S11
macp->m_margin = VLAN_TAGSZ;
#endif
macp->m_v12n = MAC_VIRT_LEVEL1;
status = mac_register(macp, &mgp->mh);
if (status != 0) {
cmn_err(CE_WARN, "%s: mac_register failed with %d\n",
mgp->name, status);
goto abort_with_info;
}
myri10ge_ndd_init(mgp);
if (myri10ge_verbose)
printf("%s: %s, tx bndry %d, fw %s\n", mgp->name,
mgp->intr_type, mgp->tx_boundary, mgp->fw_name);
mutex_enter(&myri10ge_param_lock);
mgp->next = mgp_list;
mgp_list = mgp;
mutex_exit(&myri10ge_param_lock);
kmem_free(macp, sizeof (*macp) * 8);
mac_free(omacp);
return (DDI_SUCCESS);
abort_with_info:
myri10ge_info_destroy(mgp);
abort_with_stats:
myri10ge_nic_stat_destroy(mgp);
abort_with_interrupts:
mutex_destroy(&mgp->cmd_lock);
mutex_destroy(&mgp->intrlock);
myri10ge_rem_intrs(mgp, 1);
abort_with_slices:
myri10ge_free_slices(mgp);
abort_with_dummy_rdma:
myri10ge_dummy_rdma(mgp, 0);
abort_with_mapped:
ddi_regs_map_free(&mgp->io_handle);
myri10ge_dma_free(&mgp->cmd_dma);
abort_with_mgp:
kmem_free(mgp, sizeof (*mgp));
abort_with_macinfo:
kmem_free(macp, sizeof (*macp) * 8);
mac_free(omacp);
abort_with_cfg_hdl:
pci_config_teardown(&handle);
return (DDI_FAILURE);
}
static int
myri10ge_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
struct myri10ge_priv *mgp, *tmp;
int status, i, jbufs_alloced;
if (cmd == DDI_SUSPEND) {
status = myri10ge_suspend(dip);
return (status);
}
if (cmd != DDI_DETACH) {
return (DDI_FAILURE);
}
mgp = ddi_get_driver_private(dip);
mutex_enter(&mgp->intrlock);
jbufs_alloced = 0;
for (i = 0; i < mgp->num_slices; i++) {
myri10ge_remove_jbufs(&mgp->ss[i]);
jbufs_alloced += mgp->ss[i].jpool.num_alloc;
}
mutex_exit(&mgp->intrlock);
if (jbufs_alloced != 0) {
cmn_err(CE_NOTE, "%s: %d loaned rx buffers remain\n",
mgp->name, jbufs_alloced);
return (DDI_FAILURE);
}
mutex_enter(&myri10ge_param_lock);
if (mgp->refcnt != 0) {
mutex_exit(&myri10ge_param_lock);
cmn_err(CE_NOTE, "%s: %d external refs remain\n",
mgp->name, mgp->refcnt);
return (DDI_FAILURE);
}
mutex_exit(&myri10ge_param_lock);
status = mac_unregister(mgp->mh);
if (status != DDI_SUCCESS)
return (status);
myri10ge_ndd_fini(mgp);
myri10ge_dummy_rdma(mgp, 0);
myri10ge_nic_stat_destroy(mgp);
myri10ge_info_destroy(mgp);
mutex_destroy(&mgp->cmd_lock);
mutex_destroy(&mgp->intrlock);
myri10ge_rem_intrs(mgp, 1);
myri10ge_free_slices(mgp);
ddi_regs_map_free(&mgp->io_handle);
myri10ge_dma_free(&mgp->cmd_dma);
pci_config_teardown(&mgp->cfg_hdl);
mutex_enter(&myri10ge_param_lock);
if (mgp_list == mgp) {
mgp_list = mgp->next;
} else {
tmp = mgp_list;
while (tmp->next != mgp && tmp->next != NULL)
tmp = tmp->next;
if (tmp->next != NULL)
tmp->next = tmp->next->next;
}
kmem_free(mgp, sizeof (*mgp));
mutex_exit(&myri10ge_param_lock);
return (DDI_SUCCESS);
}
static int
myri10ge_poll_down(struct myri10ge_priv *mgp)
{
struct myri10ge_slice_state *ss = mgp->ss;
mcp_irq_data_t *stats = ss->fw_stats;
int valid;
int found_down = 0;
if (! *((volatile uint8_t *)& stats->valid))
return (0);
valid = stats->valid;
if (mgp->ddi_intr_type == DDI_INTR_TYPE_FIXED) {
*mgp->irq_deassert = 0;
mb();
while (*((volatile uint8_t *)& stats->valid))
;
}
if (stats->stats_updated && stats->link_down)
found_down = 1;
if (valid & 0x1)
*ss->irq_claim = BE_32(3);
*(ss->irq_claim + 1) = BE_32(3);
return (found_down);
}
static int
myri10ge_quiesce(dev_info_t *dip)
{
struct myri10ge_priv *mgp;
myri10ge_cmd_t cmd;
int status, down, i;
mgp = ddi_get_driver_private(dip);
if (mgp == NULL)
return (DDI_FAILURE);
if (mgp->running == MYRI10GE_ETH_STOPPED)
return (DDI_SUCCESS);
status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd);
if (status) {
cmn_err(CE_WARN, "%s: Couldn't bring down link\n", mgp->name);
return (DDI_FAILURE);
}
for (i = 0; i < 20; i++) {
down = myri10ge_poll_down(mgp);
if (down)
break;
delay(drv_usectohz(100000));
mb();
}
if (down)
return (DDI_SUCCESS);
return (DDI_FAILURE);
}
static void
myri10ge_find_lastfree(void)
{
mblk_t *mp = allocb(1024, 0);
dblk_t *dbp;
if (mp == NULL) {
cmn_err(CE_WARN, "myri10ge_find_lastfree failed\n");
return;
}
dbp = mp->b_datap;
myri10ge_db_lastfree = (void *)dbp->db_lastfree;
}
int
_init(void)
{
int i;
if (myri10ge_verbose)
cmn_err(CE_NOTE,
"Myricom 10G driver (10GbE) version %s loading\n",
MYRI10GE_VERSION_STR);
myri10ge_find_lastfree();
mac_init_ops(&myri10ge_ops, "myri10ge");
mutex_init(&myri10ge_param_lock, NULL, MUTEX_DEFAULT, NULL);
if ((i = mod_install(&modlinkage)) != 0) {
cmn_err(CE_WARN, "mod_install returned %d\n", i);
mac_fini_ops(&myri10ge_ops);
mutex_destroy(&myri10ge_param_lock);
}
return (i);
}
int
_fini(void)
{
int i;
i = mod_remove(&modlinkage);
if (i != 0) {
return (i);
}
mac_fini_ops(&myri10ge_ops);
mutex_destroy(&myri10ge_param_lock);
return (0);
}
int
_info(struct modinfo *modinfop)
{
return (mod_info(&modlinkage, modinfop));
}
