/*
 * CDDL HEADER START
 *
 * Copyright(c) 2007-2009 Intel Corporation. All rights reserved.
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at:
 *      http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When using or redistributing this file, you may do so under the
 * License only. No other modification of this header is permitted.
 *
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include "ixgbe_sw.h"

static int ixgbe_alloc_tbd_ring(ixgbe_tx_ring_t *);
static void ixgbe_free_tbd_ring(ixgbe_tx_ring_t *);
static int ixgbe_alloc_rbd_ring(ixgbe_rx_data_t *);
static void ixgbe_free_rbd_ring(ixgbe_rx_data_t *);
static int ixgbe_alloc_dma_buffer(ixgbe_t *, dma_buffer_t *, size_t);
static int ixgbe_alloc_tcb_lists(ixgbe_tx_ring_t *);
static void ixgbe_free_tcb_lists(ixgbe_tx_ring_t *);
static int ixgbe_alloc_rcb_lists(ixgbe_rx_data_t *);
static void ixgbe_free_rcb_lists(ixgbe_rx_data_t *);

#ifdef __sparc
#define IXGBE_DMA_ALIGNMENT     0x0000000000002000ull
#else
#define IXGBE_DMA_ALIGNMENT     0x0000000000001000ull
#endif

/*
 * DMA attributes for tx/rx descriptors.
 */
static ddi_dma_attr_t ixgbe_desc_dma_attr = {
        DMA_ATTR_V0,                    /* version number */
        0x0000000000000000ull,          /* low address */
        0xFFFFFFFFFFFFFFFFull,          /* high address */
        0x00000000FFFFFFFFull,          /* dma counter max */
        IXGBE_DMA_ALIGNMENT,            /* alignment */
        0x00000FFF,                     /* burst sizes */
        0x00000001,                     /* minimum transfer size */
        0x00000000FFFFFFFFull,          /* maximum transfer size */
        0xFFFFFFFFFFFFFFFFull,          /* maximum segment size */
        1,                              /* scatter/gather list length */
        0x00000001,                     /* granularity */
        DDI_DMA_FLAGERR                 /* DMA flags */
};

/*
 * DMA attributes for tx/rx buffers.
 */
static ddi_dma_attr_t ixgbe_buf_dma_attr = {
        DMA_ATTR_V0,                    /* version number */
        0x0000000000000000ull,          /* low address */
        0xFFFFFFFFFFFFFFFFull,          /* high address */
        0x00000000FFFFFFFFull,          /* dma counter max */
        IXGBE_DMA_ALIGNMENT,            /* alignment */
        0x00000FFF,                     /* burst sizes */
        0x00000001,                     /* minimum transfer size */
        0x00000000FFFFFFFFull,          /* maximum transfer size */
        0xFFFFFFFFFFFFFFFFull,          /* maximum segment size  */
        1,                              /* scatter/gather list length */
        0x00000001,                     /* granularity */
        DDI_DMA_FLAGERR                 /* DMA flags */
};

/*
 * DMA attributes for transmit.
 */
static ddi_dma_attr_t ixgbe_tx_dma_attr = {
        DMA_ATTR_V0,                    /* version number */
        0x0000000000000000ull,          /* low address */
        0xFFFFFFFFFFFFFFFFull,          /* high address */
        0x00000000FFFFFFFFull,          /* dma counter max */
        1,                              /* alignment */
        0x00000FFF,                     /* burst sizes */
        0x00000001,                     /* minimum transfer size */
        0x00000000FFFFFFFFull,          /* maximum transfer size */
        0xFFFFFFFFFFFFFFFFull,          /* maximum segment size  */
        MAX_COOKIE,                     /* scatter/gather list length */
        0x00000001,                     /* granularity */
        DDI_DMA_FLAGERR                 /* DMA flags */
};

/*
 * DMA access attributes for descriptors.
 */
static ddi_device_acc_attr_t ixgbe_desc_acc_attr = {
        DDI_DEVICE_ATTR_V0,
        DDI_STRUCTURE_LE_ACC,
        DDI_STRICTORDER_ACC
};

/*
 * DMA access attributes for buffers.
 */
static ddi_device_acc_attr_t ixgbe_buf_acc_attr = {
        DDI_DEVICE_ATTR_V0,
        DDI_NEVERSWAP_ACC,
        DDI_STRICTORDER_ACC
};

/*
 * ixgbe_alloc_dma - Allocate DMA resources for all rx/tx rings.
 */
int
ixgbe_alloc_dma(ixgbe_t *ixgbe)
{
        ixgbe_rx_ring_t *rx_ring;
        ixgbe_rx_data_t *rx_data;
        ixgbe_tx_ring_t *tx_ring;
        int i;

        for (i = 0; i < ixgbe->num_rx_rings; i++) {
                /*
                 * Allocate receive desciptor ring and control block lists
                 */
                rx_ring = &ixgbe->rx_rings[i];
                rx_data = rx_ring->rx_data;

                if (ixgbe_alloc_rbd_ring(rx_data) != IXGBE_SUCCESS)
                        goto alloc_dma_failure;

                if (ixgbe_alloc_rcb_lists(rx_data) != IXGBE_SUCCESS)
                        goto alloc_dma_failure;
        }

        for (i = 0; i < ixgbe->num_tx_rings; i++) {
                /*
                 * Allocate transmit desciptor ring and control block lists
                 */
                tx_ring = &ixgbe->tx_rings[i];

                if (ixgbe_alloc_tbd_ring(tx_ring) != IXGBE_SUCCESS)
                        goto alloc_dma_failure;

                if (ixgbe_alloc_tcb_lists(tx_ring) != IXGBE_SUCCESS)
                        goto alloc_dma_failure;
        }

        return (IXGBE_SUCCESS);

alloc_dma_failure:
        ixgbe_free_dma(ixgbe);

        return (IXGBE_FAILURE);
}

/*
 * ixgbe_free_dma - Free all the DMA resources of all rx/tx rings.
 */
void
ixgbe_free_dma(ixgbe_t *ixgbe)
{
        ixgbe_rx_ring_t *rx_ring;
        ixgbe_rx_data_t *rx_data;
        ixgbe_tx_ring_t *tx_ring;
        int i;

        /*
         * Free DMA resources of rx rings
         */
        for (i = 0; i < ixgbe->num_rx_rings; i++) {
                rx_ring = &ixgbe->rx_rings[i];
                rx_data = rx_ring->rx_data;

                ixgbe_free_rbd_ring(rx_data);
                ixgbe_free_rcb_lists(rx_data);
        }

        /*
         * Free DMA resources of tx rings
         */
        for (i = 0; i < ixgbe->num_tx_rings; i++) {
                tx_ring = &ixgbe->tx_rings[i];
                ixgbe_free_tbd_ring(tx_ring);
                ixgbe_free_tcb_lists(tx_ring);
        }
}

int
ixgbe_alloc_rx_ring_data(ixgbe_rx_ring_t *rx_ring)
{
        ixgbe_rx_data_t *rx_data;
        ixgbe_t *ixgbe = rx_ring->ixgbe;
        uint32_t rcb_count;

        /*
         * Allocate memory for software receive rings
         */
        rx_data = kmem_zalloc(sizeof (ixgbe_rx_data_t), KM_NOSLEEP);

        if (rx_data == NULL) {
                ixgbe_error(ixgbe, "Allocate software receive rings failed");
                return (IXGBE_FAILURE);
        }

        rx_data->rx_ring = rx_ring;
        mutex_init(&rx_data->recycle_lock, NULL,
            MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));

        rx_data->ring_size = ixgbe->rx_ring_size;
        rx_data->free_list_size = ixgbe->rx_ring_size;

        rx_data->rcb_head = 0;
        rx_data->rcb_tail = 0;
        rx_data->rcb_free = rx_data->free_list_size;

        /*
         * Allocate memory for the work list.
         */
        rx_data->work_list = kmem_zalloc(sizeof (rx_control_block_t *) *
            rx_data->ring_size, KM_NOSLEEP);

        if (rx_data->work_list == NULL) {
                ixgbe_error(ixgbe,
                    "Could not allocate memory for rx work list");
                goto alloc_rx_data_failure;
        }

        /*
         * Allocate memory for the free list.
         */
        rx_data->free_list = kmem_zalloc(sizeof (rx_control_block_t *) *
            rx_data->free_list_size, KM_NOSLEEP);

        if (rx_data->free_list == NULL) {
                ixgbe_error(ixgbe,
                    "Cound not allocate memory for rx free list");
                goto alloc_rx_data_failure;
        }

        /*
         * Allocate memory for the rx control blocks for work list and
         * free list.
         */
        rcb_count = rx_data->ring_size + rx_data->free_list_size;
        rx_data->rcb_area =
            kmem_zalloc(sizeof (rx_control_block_t) * rcb_count,
            KM_NOSLEEP);

        if (rx_data->rcb_area == NULL) {
                ixgbe_error(ixgbe,
                    "Cound not allocate memory for rx control blocks");
                goto alloc_rx_data_failure;
        }

        rx_ring->rx_data = rx_data;
        return (IXGBE_SUCCESS);

alloc_rx_data_failure:
        ixgbe_free_rx_ring_data(rx_data);
        return (IXGBE_FAILURE);
}

void
ixgbe_free_rx_ring_data(ixgbe_rx_data_t *rx_data)
{
        uint32_t rcb_count;

        if (rx_data == NULL)
                return;

        ASSERT(rx_data->rcb_pending == 0);

        rcb_count = rx_data->ring_size + rx_data->free_list_size;
        if (rx_data->rcb_area != NULL) {
                kmem_free(rx_data->rcb_area,
                    sizeof (rx_control_block_t) * rcb_count);
                rx_data->rcb_area = NULL;
        }

        if (rx_data->work_list != NULL) {
                kmem_free(rx_data->work_list,
                    sizeof (rx_control_block_t *) * rx_data->ring_size);
                rx_data->work_list = NULL;
        }

        if (rx_data->free_list != NULL) {
                kmem_free(rx_data->free_list,
                    sizeof (rx_control_block_t *) * rx_data->free_list_size);
                rx_data->free_list = NULL;
        }

        mutex_destroy(&rx_data->recycle_lock);
        kmem_free(rx_data, sizeof (ixgbe_rx_data_t));
}

/*
 * ixgbe_alloc_tbd_ring - Memory allocation for the tx descriptors of one ring.
 */
static int
ixgbe_alloc_tbd_ring(ixgbe_tx_ring_t *tx_ring)
{
        int ret;
        size_t size;
        size_t len;
        uint_t cookie_num;
        dev_info_t *devinfo;
        ddi_dma_cookie_t cookie;
        ixgbe_t *ixgbe = tx_ring->ixgbe;

        devinfo = ixgbe->dip;
        size = sizeof (union ixgbe_adv_tx_desc) * tx_ring->ring_size;

        /*
         * If tx head write-back is enabled, an extra tbd is allocated
         * to save the head write-back value
         */
        if (ixgbe->tx_head_wb_enable) {
                size += sizeof (union ixgbe_adv_tx_desc);
        }

        /*
         * Allocate a DMA handle for the transmit descriptor
         * memory area.
         */
        ret = ddi_dma_alloc_handle(devinfo, &ixgbe_desc_dma_attr,
            DDI_DMA_DONTWAIT, NULL,
            &tx_ring->tbd_area.dma_handle);

        if (ret != DDI_SUCCESS) {
                ixgbe_error(ixgbe,
                    "Could not allocate tbd dma handle: %x", ret);
                tx_ring->tbd_area.dma_handle = NULL;

                return (IXGBE_FAILURE);
        }

        /*
         * Allocate memory to DMA data to and from the transmit
         * descriptors.
         */
        ret = ddi_dma_mem_alloc(tx_ring->tbd_area.dma_handle,
            size, &ixgbe_desc_acc_attr, DDI_DMA_CONSISTENT,
            DDI_DMA_DONTWAIT, NULL,
            (caddr_t *)&tx_ring->tbd_area.address,
            &len, &tx_ring->tbd_area.acc_handle);

        if (ret != DDI_SUCCESS) {
                ixgbe_error(ixgbe,
                    "Could not allocate tbd dma memory: %x", ret);
                tx_ring->tbd_area.acc_handle = NULL;
                tx_ring->tbd_area.address = NULL;
                if (tx_ring->tbd_area.dma_handle != NULL) {
                        ddi_dma_free_handle(&tx_ring->tbd_area.dma_handle);
                        tx_ring->tbd_area.dma_handle = NULL;
                }
                return (IXGBE_FAILURE);
        }

        /*
         * Initialize the entire transmit buffer descriptor area to zero
         */
        bzero(tx_ring->tbd_area.address, len);

        /*
         * Allocates DMA resources for the memory that was allocated by
         * the ddi_dma_mem_alloc call. The DMA resources then get bound to the
         * the memory address
         */
        ret = ddi_dma_addr_bind_handle(tx_ring->tbd_area.dma_handle,
            NULL, (caddr_t)tx_ring->tbd_area.address,
            len, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
            DDI_DMA_DONTWAIT, NULL, &cookie, &cookie_num);

        if (ret != DDI_DMA_MAPPED) {
                ixgbe_error(ixgbe,
                    "Could not bind tbd dma resource: %x", ret);
                tx_ring->tbd_area.dma_address = 0;
                if (tx_ring->tbd_area.acc_handle != NULL) {
                        ddi_dma_mem_free(&tx_ring->tbd_area.acc_handle);
                        tx_ring->tbd_area.acc_handle = NULL;
                        tx_ring->tbd_area.address = NULL;
                }
                if (tx_ring->tbd_area.dma_handle != NULL) {
                        ddi_dma_free_handle(&tx_ring->tbd_area.dma_handle);
                        tx_ring->tbd_area.dma_handle = NULL;
                }
                return (IXGBE_FAILURE);
        }

        ASSERT(cookie_num == 1);

        tx_ring->tbd_area.dma_address = cookie.dmac_laddress;
        tx_ring->tbd_area.size = len;

        tx_ring->tbd_ring = (union ixgbe_adv_tx_desc *)(uintptr_t)
            tx_ring->tbd_area.address;

        return (IXGBE_SUCCESS);
}

/*
 * ixgbe_free_tbd_ring - Free the tx descriptors of one ring.
 */
static void
ixgbe_free_tbd_ring(ixgbe_tx_ring_t *tx_ring)
{
        if (tx_ring->tbd_area.dma_handle != NULL) {
                (void) ddi_dma_unbind_handle(tx_ring->tbd_area.dma_handle);
        }
        if (tx_ring->tbd_area.acc_handle != NULL) {
                ddi_dma_mem_free(&tx_ring->tbd_area.acc_handle);
                tx_ring->tbd_area.acc_handle = NULL;
        }
        if (tx_ring->tbd_area.dma_handle != NULL) {
                ddi_dma_free_handle(&tx_ring->tbd_area.dma_handle);
                tx_ring->tbd_area.dma_handle = NULL;
        }
        tx_ring->tbd_area.address = NULL;
        tx_ring->tbd_area.dma_address = 0;
        tx_ring->tbd_area.size = 0;

        tx_ring->tbd_ring = NULL;
}

/*
 * ixgbe_alloc_rbd_ring - Memory allocation for the rx descriptors of one ring.
 */
static int
ixgbe_alloc_rbd_ring(ixgbe_rx_data_t *rx_data)
{
        int ret;
        size_t size;
        size_t len;
        uint_t cookie_num;
        dev_info_t *devinfo;
        ddi_dma_cookie_t cookie;
        ixgbe_t *ixgbe = rx_data->rx_ring->ixgbe;

        devinfo = ixgbe->dip;
        size = sizeof (union ixgbe_adv_rx_desc) * rx_data->ring_size;

        /*
         * Allocate a new DMA handle for the receive descriptor
         * memory area.
         */
        ret = ddi_dma_alloc_handle(devinfo, &ixgbe_desc_dma_attr,
            DDI_DMA_DONTWAIT, NULL,
            &rx_data->rbd_area.dma_handle);

        if (ret != DDI_SUCCESS) {
                ixgbe_error(ixgbe,
                    "Could not allocate rbd dma handle: %x", ret);
                rx_data->rbd_area.dma_handle = NULL;
                return (IXGBE_FAILURE);
        }

        /*
         * Allocate memory to DMA data to and from the receive
         * descriptors.
         */
        ret = ddi_dma_mem_alloc(rx_data->rbd_area.dma_handle,
            size, &ixgbe_desc_acc_attr, DDI_DMA_CONSISTENT,
            DDI_DMA_DONTWAIT, NULL,
            (caddr_t *)&rx_data->rbd_area.address,
            &len, &rx_data->rbd_area.acc_handle);

        if (ret != DDI_SUCCESS) {
                ixgbe_error(ixgbe,
                    "Could not allocate rbd dma memory: %x", ret);
                rx_data->rbd_area.acc_handle = NULL;
                rx_data->rbd_area.address = NULL;
                if (rx_data->rbd_area.dma_handle != NULL) {
                        ddi_dma_free_handle(&rx_data->rbd_area.dma_handle);
                        rx_data->rbd_area.dma_handle = NULL;
                }
                return (IXGBE_FAILURE);
        }

        /*
         * Initialize the entire transmit buffer descriptor area to zero
         */
        bzero(rx_data->rbd_area.address, len);

        /*
         * Allocates DMA resources for the memory that was allocated by
         * the ddi_dma_mem_alloc call.
         */
        ret = ddi_dma_addr_bind_handle(rx_data->rbd_area.dma_handle,
            NULL, (caddr_t)rx_data->rbd_area.address,
            len, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
            DDI_DMA_DONTWAIT, NULL, &cookie, &cookie_num);

        if (ret != DDI_DMA_MAPPED) {
                ixgbe_error(ixgbe,
                    "Could not bind rbd dma resource: %x", ret);
                rx_data->rbd_area.dma_address = 0;
                if (rx_data->rbd_area.acc_handle != NULL) {
                        ddi_dma_mem_free(&rx_data->rbd_area.acc_handle);
                        rx_data->rbd_area.acc_handle = NULL;
                        rx_data->rbd_area.address = NULL;
                }
                if (rx_data->rbd_area.dma_handle != NULL) {
                        ddi_dma_free_handle(&rx_data->rbd_area.dma_handle);
                        rx_data->rbd_area.dma_handle = NULL;
                }
                return (IXGBE_FAILURE);
        }

        ASSERT(cookie_num == 1);

        rx_data->rbd_area.dma_address = cookie.dmac_laddress;
        rx_data->rbd_area.size = len;

        rx_data->rbd_ring = (union ixgbe_adv_rx_desc *)(uintptr_t)
            rx_data->rbd_area.address;

        return (IXGBE_SUCCESS);
}

/*
 * ixgbe_free_rbd_ring - Free the rx descriptors of one ring.
 */
static void
ixgbe_free_rbd_ring(ixgbe_rx_data_t *rx_data)
{
        if (rx_data->rbd_area.dma_handle != NULL) {
                (void) ddi_dma_unbind_handle(rx_data->rbd_area.dma_handle);
        }
        if (rx_data->rbd_area.acc_handle != NULL) {
                ddi_dma_mem_free(&rx_data->rbd_area.acc_handle);
                rx_data->rbd_area.acc_handle = NULL;
        }
        if (rx_data->rbd_area.dma_handle != NULL) {
                ddi_dma_free_handle(&rx_data->rbd_area.dma_handle);
                rx_data->rbd_area.dma_handle = NULL;
        }
        rx_data->rbd_area.address = NULL;
        rx_data->rbd_area.dma_address = 0;
        rx_data->rbd_area.size = 0;

        rx_data->rbd_ring = NULL;
}

/*
 * ixgbe_alloc_dma_buffer - Allocate DMA resources for a DMA buffer.
 */
static int
ixgbe_alloc_dma_buffer(ixgbe_t *ixgbe, dma_buffer_t *buf, size_t size)
{
        int ret;
        dev_info_t *devinfo = ixgbe->dip;
        ddi_dma_cookie_t cookie;
        size_t len;
        uint_t cookie_num;

        ret = ddi_dma_alloc_handle(devinfo,
            &ixgbe_buf_dma_attr, DDI_DMA_DONTWAIT,
            NULL, &buf->dma_handle);

        if (ret != DDI_SUCCESS) {
                buf->dma_handle = NULL;
                ixgbe_error(ixgbe,
                    "Could not allocate dma buffer handle: %x", ret);
                return (IXGBE_FAILURE);
        }

        ret = ddi_dma_mem_alloc(buf->dma_handle,
            size, &ixgbe_buf_acc_attr, DDI_DMA_STREAMING,
            DDI_DMA_DONTWAIT, NULL, &buf->address,
            &len, &buf->acc_handle);

        if (ret != DDI_SUCCESS) {
                buf->acc_handle = NULL;
                buf->address = NULL;
                if (buf->dma_handle != NULL) {
                        ddi_dma_free_handle(&buf->dma_handle);
                        buf->dma_handle = NULL;
                }
                ixgbe_error(ixgbe,
                    "Could not allocate dma buffer memory: %x", ret);
                return (IXGBE_FAILURE);
        }

        ret = ddi_dma_addr_bind_handle(buf->dma_handle, NULL,
            buf->address,
            len, DDI_DMA_RDWR | DDI_DMA_STREAMING,
            DDI_DMA_DONTWAIT, NULL, &cookie, &cookie_num);

        if (ret != DDI_DMA_MAPPED) {
                buf->dma_address = 0;
                if (buf->acc_handle != NULL) {
                        ddi_dma_mem_free(&buf->acc_handle);
                        buf->acc_handle = NULL;
                        buf->address = NULL;
                }
                if (buf->dma_handle != NULL) {
                        ddi_dma_free_handle(&buf->dma_handle);
                        buf->dma_handle = NULL;
                }
                ixgbe_error(ixgbe,
                    "Could not bind dma buffer handle: %x", ret);
                return (IXGBE_FAILURE);
        }

        ASSERT(cookie_num == 1);

        buf->dma_address = cookie.dmac_laddress;
        buf->size = len;
        buf->len = 0;

        return (IXGBE_SUCCESS);
}

/*
 * ixgbe_free_dma_buffer - Free one allocated area of dma memory and handle.
 */
void
ixgbe_free_dma_buffer(dma_buffer_t *buf)
{
        if (buf->dma_handle != NULL) {
                (void) ddi_dma_unbind_handle(buf->dma_handle);
                buf->dma_address = 0;
        } else {
                return;
        }

        if (buf->acc_handle != NULL) {
                ddi_dma_mem_free(&buf->acc_handle);
                buf->acc_handle = NULL;
                buf->address = NULL;
        }

        if (buf->dma_handle != NULL) {
                ddi_dma_free_handle(&buf->dma_handle);
                buf->dma_handle = NULL;
        }

        buf->size = 0;
        buf->len = 0;
}

/*
 * ixgbe_alloc_tcb_lists - Memory allocation for the transmit control bolcks
 * of one ring.
 */
static int
ixgbe_alloc_tcb_lists(ixgbe_tx_ring_t *tx_ring)
{
        int i;
        int ret;
        tx_control_block_t *tcb;
        dma_buffer_t *tx_buf;
        ixgbe_t *ixgbe = tx_ring->ixgbe;
        dev_info_t *devinfo = ixgbe->dip;

        /*
         * Allocate memory for the work list.
         */
        tx_ring->work_list = kmem_zalloc(sizeof (tx_control_block_t *) *
            tx_ring->ring_size, KM_NOSLEEP);

        if (tx_ring->work_list == NULL) {
                ixgbe_error(ixgbe,
                    "Cound not allocate memory for tx work list");
                return (IXGBE_FAILURE);
        }

        /*
         * Allocate memory for the free list.
         */
        tx_ring->free_list = kmem_zalloc(sizeof (tx_control_block_t *) *
            tx_ring->free_list_size, KM_NOSLEEP);

        if (tx_ring->free_list == NULL) {
                kmem_free(tx_ring->work_list,
                    sizeof (tx_control_block_t *) * tx_ring->ring_size);
                tx_ring->work_list = NULL;

                ixgbe_error(ixgbe,
                    "Cound not allocate memory for tx free list");
                return (IXGBE_FAILURE);
        }

        /*
         * Allocate memory for the tx control blocks of free list.
         */
        tx_ring->tcb_area =
            kmem_zalloc(sizeof (tx_control_block_t) *
            tx_ring->free_list_size, KM_NOSLEEP);

        if (tx_ring->tcb_area == NULL) {
                kmem_free(tx_ring->work_list,
                    sizeof (tx_control_block_t *) * tx_ring->ring_size);
                tx_ring->work_list = NULL;

                kmem_free(tx_ring->free_list,
                    sizeof (tx_control_block_t *) * tx_ring->free_list_size);
                tx_ring->free_list = NULL;

                ixgbe_error(ixgbe,
                    "Cound not allocate memory for tx control blocks");
                return (IXGBE_FAILURE);
        }

        /*
         * Allocate dma memory for the tx control block of free list.
         */
        tcb = tx_ring->tcb_area;
        for (i = 0; i < tx_ring->free_list_size; i++, tcb++) {
                ASSERT(tcb != NULL);

                tx_ring->free_list[i] = tcb;

                /*
                 * Pre-allocate dma handles for transmit. These dma handles
                 * will be dynamically bound to the data buffers passed down
                 * from the upper layers at the time of transmitting.
                 */
                ret = ddi_dma_alloc_handle(devinfo,
                    &ixgbe_tx_dma_attr,
                    DDI_DMA_DONTWAIT, NULL,
                    &tcb->tx_dma_handle);
                if (ret != DDI_SUCCESS) {
                        tcb->tx_dma_handle = NULL;
                        ixgbe_error(ixgbe,
                            "Could not allocate tx dma handle: %x", ret);
                        goto alloc_tcb_lists_fail;
                }

                /*
                 * Pre-allocate transmit buffers for packets that the
                 * size is less than bcopy_thresh.
                 */
                tx_buf = &tcb->tx_buf;

                ret = ixgbe_alloc_dma_buffer(ixgbe,
                    tx_buf, ixgbe->tx_buf_size);

                if (ret != IXGBE_SUCCESS) {
                        ASSERT(tcb->tx_dma_handle != NULL);
                        ddi_dma_free_handle(&tcb->tx_dma_handle);
                        tcb->tx_dma_handle = NULL;
                        ixgbe_error(ixgbe, "Allocate tx dma buffer failed");
                        goto alloc_tcb_lists_fail;
                }

                tcb->last_index = MAX_TX_RING_SIZE;
        }

        return (IXGBE_SUCCESS);

alloc_tcb_lists_fail:
        ixgbe_free_tcb_lists(tx_ring);

        return (IXGBE_FAILURE);
}

/*
 * ixgbe_free_tcb_lists - Release the memory allocated for
 * the transmit control bolcks of one ring.
 */
static void
ixgbe_free_tcb_lists(ixgbe_tx_ring_t *tx_ring)
{
        int i;
        tx_control_block_t *tcb;

        tcb = tx_ring->tcb_area;
        if (tcb == NULL)
                return;

        for (i = 0; i < tx_ring->free_list_size; i++, tcb++) {
                ASSERT(tcb != NULL);

                /* Free the tx dma handle for dynamical binding */
                if (tcb->tx_dma_handle != NULL) {
                        ddi_dma_free_handle(&tcb->tx_dma_handle);
                        tcb->tx_dma_handle = NULL;
                } else {
                        /*
                         * If the dma handle is NULL, then we don't
                         * have to check the remaining.
                         */
                        break;
                }

                ixgbe_free_dma_buffer(&tcb->tx_buf);
        }

        if (tx_ring->tcb_area != NULL) {
                kmem_free(tx_ring->tcb_area,
                    sizeof (tx_control_block_t) * tx_ring->free_list_size);
                tx_ring->tcb_area = NULL;
        }

        if (tx_ring->work_list != NULL) {
                kmem_free(tx_ring->work_list,
                    sizeof (tx_control_block_t *) * tx_ring->ring_size);
                tx_ring->work_list = NULL;
        }

        if (tx_ring->free_list != NULL) {
                kmem_free(tx_ring->free_list,
                    sizeof (tx_control_block_t *) * tx_ring->free_list_size);
                tx_ring->free_list = NULL;
        }
}

/*
 * ixgbe_alloc_rcb_lists - Memory allocation for the receive control blocks
 * of one ring.
 */
static int
ixgbe_alloc_rcb_lists(ixgbe_rx_data_t *rx_data)
{
        int i;
        int ret;
        rx_control_block_t *rcb;
        ixgbe_t *ixgbe = rx_data->rx_ring->ixgbe;
        dma_buffer_t *rx_buf;
        uint32_t rcb_count;

        /*
         * Allocate memory for the rx control blocks for work list and
         * free list.
         */
        rcb_count = rx_data->ring_size + rx_data->free_list_size;
        rcb = rx_data->rcb_area;

        for (i = 0; i < rcb_count; i++, rcb++) {
                ASSERT(rcb != NULL);

                if (i < rx_data->ring_size) {
                        /* Attach the rx control block to the work list */
                        rx_data->work_list[i] = rcb;
                } else {
                        /* Attach the rx control block to the free list */
                        rx_data->free_list[i - rx_data->ring_size] = rcb;
                }

                rx_buf = &rcb->rx_buf;
                ret = ixgbe_alloc_dma_buffer(ixgbe,
                    rx_buf, ixgbe->rx_buf_size);

                if (ret != IXGBE_SUCCESS) {
                        ixgbe_error(ixgbe, "Allocate rx dma buffer failed");
                        goto alloc_rcb_lists_fail;
                }

                rx_buf->size -= IPHDR_ALIGN_ROOM;
                rx_buf->address += IPHDR_ALIGN_ROOM;
                rx_buf->dma_address += IPHDR_ALIGN_ROOM;

                rcb->ref_cnt = 1;
                rcb->rx_data = (ixgbe_rx_data_t *)rx_data;
                rcb->free_rtn.free_func = ixgbe_rx_recycle;
                rcb->free_rtn.free_arg = (char *)rcb;
                rcb->lro_prev = -1;
                rcb->lro_next = -1;
                rcb->lro_pkt = B_FALSE;
                rcb->mp = desballoc((unsigned char *)
                    rx_buf->address,
                    rx_buf->size,
                    0, &rcb->free_rtn);
        }

        return (IXGBE_SUCCESS);

alloc_rcb_lists_fail:
        ixgbe_free_rcb_lists(rx_data);

        return (IXGBE_FAILURE);
}

/*
 * ixgbe_free_rcb_lists - Free the receive control blocks of one ring.
 */
static void
ixgbe_free_rcb_lists(ixgbe_rx_data_t *rx_data)
{
        ixgbe_t *ixgbe;
        rx_control_block_t *rcb;
        uint32_t rcb_count;
        uint32_t ref_cnt;
        int i;

        ixgbe = rx_data->rx_ring->ixgbe;

        mutex_enter(&ixgbe->rx_pending_lock);

        rcb = rx_data->rcb_area;
        rcb_count = rx_data->ring_size + rx_data->free_list_size;

        for (i = 0; i < rcb_count; i++, rcb++) {
                ASSERT(rcb != NULL);

                ref_cnt = atomic_dec_32_nv(&rcb->ref_cnt);
                if (ref_cnt == 0) {
                        if (rcb->mp != NULL) {
                                freemsg(rcb->mp);
                                rcb->mp = NULL;
                        }
                        ixgbe_free_dma_buffer(&rcb->rx_buf);
                } else {
                        atomic_inc_32(&rx_data->rcb_pending);
                        atomic_inc_32(&ixgbe->rcb_pending);
                }
        }

        mutex_exit(&ixgbe->rx_pending_lock);
}

/*
 * ixgbe_set_fma_flags - Set the attribute for fma support.
 */
void
ixgbe_set_fma_flags(int dma_flag)
{
        if (dma_flag) {
                ixgbe_tx_dma_attr.dma_attr_flags = DDI_DMA_FLAGERR;
                ixgbe_buf_dma_attr.dma_attr_flags = DDI_DMA_FLAGERR;
                ixgbe_desc_dma_attr.dma_attr_flags = DDI_DMA_FLAGERR;
        } else {
                ixgbe_tx_dma_attr.dma_attr_flags = 0;
                ixgbe_buf_dma_attr.dma_attr_flags = 0;
                ixgbe_desc_dma_attr.dma_attr_flags = 0;
        }
}