/*
 * Copyright (c) 2010 Broadcom Corporation
 *
 * Permission to use, copy, modify, and/or distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <linux/slab.h>
#include <linux/delay.h>
#include <linux/pci.h>
#include <net/cfg80211.h>
#include <net/mac80211.h>

#include <brcmu_utils.h>
#include <aiutils.h>
#include "types.h"
#include "main.h"
#include "dma.h"
#include "soc.h"
#include "scb.h"
#include "ampdu.h"
#include "debug.h"
#include "brcms_trace_events.h"

/*
 * dma register field offset calculation
 */
#define DMA64REGOFFS(field)             offsetof(struct dma64regs, field)
#define DMA64TXREGOFFS(di, field)       (di->d64txregbase + DMA64REGOFFS(field))
#define DMA64RXREGOFFS(di, field)       (di->d64rxregbase + DMA64REGOFFS(field))

/*
 * DMA hardware requires each descriptor ring to be 8kB aligned, and fit within
 * a contiguous 8kB physical address.
 */
#define D64RINGALIGN_BITS       13
#define D64MAXRINGSZ            (1 << D64RINGALIGN_BITS)
#define D64RINGALIGN            (1 << D64RINGALIGN_BITS)

#define D64MAXDD        (D64MAXRINGSZ / sizeof(struct dma64desc))

/* transmit channel control */
#define D64_XC_XE               0x00000001      /* transmit enable */
#define D64_XC_SE               0x00000002      /* transmit suspend request */
#define D64_XC_LE               0x00000004      /* loopback enable */
#define D64_XC_FL               0x00000010      /* flush request */
#define D64_XC_PD               0x00000800      /* parity check disable */
#define D64_XC_AE               0x00030000      /* address extension bits */
#define D64_XC_AE_SHIFT         16

/* transmit descriptor table pointer */
#define D64_XP_LD_MASK          0x00000fff      /* last valid descriptor */

/* transmit channel status */
#define D64_XS0_CD_MASK         0x00001fff      /* current descriptor pointer */
#define D64_XS0_XS_MASK         0xf0000000      /* transmit state */
#define D64_XS0_XS_SHIFT                28
#define D64_XS0_XS_DISABLED     0x00000000      /* disabled */
#define D64_XS0_XS_ACTIVE       0x10000000      /* active */
#define D64_XS0_XS_IDLE         0x20000000      /* idle wait */
#define D64_XS0_XS_STOPPED      0x30000000      /* stopped */
#define D64_XS0_XS_SUSP         0x40000000      /* suspend pending */

#define D64_XS1_AD_MASK         0x00001fff      /* active descriptor */
#define D64_XS1_XE_MASK         0xf0000000      /* transmit errors */
#define D64_XS1_XE_SHIFT                28
#define D64_XS1_XE_NOERR        0x00000000      /* no error */
#define D64_XS1_XE_DPE          0x10000000      /* descriptor protocol error */
#define D64_XS1_XE_DFU          0x20000000      /* data fifo underrun */
#define D64_XS1_XE_DTE          0x30000000      /* data transfer error */
#define D64_XS1_XE_DESRE        0x40000000      /* descriptor read error */
#define D64_XS1_XE_COREE        0x50000000      /* core error */

/* receive channel control */
/* receive enable */
#define D64_RC_RE               0x00000001
/* receive frame offset */
#define D64_RC_RO_MASK          0x000000fe
#define D64_RC_RO_SHIFT         1
/* direct fifo receive (pio) mode */
#define D64_RC_FM               0x00000100
/* separate rx header descriptor enable */
#define D64_RC_SH               0x00000200
/* overflow continue */
#define D64_RC_OC               0x00000400
/* parity check disable */
#define D64_RC_PD               0x00000800
/* address extension bits */
#define D64_RC_AE               0x00030000
#define D64_RC_AE_SHIFT         16

/* flags for dma controller */
/* partity enable */
#define DMA_CTRL_PEN            (1 << 0)
/* rx overflow continue */
#define DMA_CTRL_ROC            (1 << 1)
/* allow rx scatter to multiple descriptors */
#define DMA_CTRL_RXMULTI        (1 << 2)
/* Unframed Rx/Tx data */
#define DMA_CTRL_UNFRAMED       (1 << 3)

/* receive descriptor table pointer */
#define D64_RP_LD_MASK          0x00000fff      /* last valid descriptor */

/* receive channel status */
#define D64_RS0_CD_MASK         0x00001fff      /* current descriptor pointer */
#define D64_RS0_RS_MASK         0xf0000000      /* receive state */
#define D64_RS0_RS_SHIFT                28
#define D64_RS0_RS_DISABLED     0x00000000      /* disabled */
#define D64_RS0_RS_ACTIVE       0x10000000      /* active */
#define D64_RS0_RS_IDLE         0x20000000      /* idle wait */
#define D64_RS0_RS_STOPPED      0x30000000      /* stopped */
#define D64_RS0_RS_SUSP         0x40000000      /* suspend pending */

#define D64_RS1_AD_MASK         0x0001ffff      /* active descriptor */
#define D64_RS1_RE_MASK         0xf0000000      /* receive errors */
#define D64_RS1_RE_SHIFT                28
#define D64_RS1_RE_NOERR        0x00000000      /* no error */
#define D64_RS1_RE_DPO          0x10000000      /* descriptor protocol error */
#define D64_RS1_RE_DFU          0x20000000      /* data fifo overflow */
#define D64_RS1_RE_DTE          0x30000000      /* data transfer error */
#define D64_RS1_RE_DESRE        0x40000000      /* descriptor read error */
#define D64_RS1_RE_COREE        0x50000000      /* core error */

/* fifoaddr */
#define D64_FA_OFF_MASK         0xffff  /* offset */
#define D64_FA_SEL_MASK         0xf0000 /* select */
#define D64_FA_SEL_SHIFT        16
#define D64_FA_SEL_XDD          0x00000 /* transmit dma data */
#define D64_FA_SEL_XDP          0x10000 /* transmit dma pointers */
#define D64_FA_SEL_RDD          0x40000 /* receive dma data */
#define D64_FA_SEL_RDP          0x50000 /* receive dma pointers */
#define D64_FA_SEL_XFD          0x80000 /* transmit fifo data */
#define D64_FA_SEL_XFP          0x90000 /* transmit fifo pointers */
#define D64_FA_SEL_RFD          0xc0000 /* receive fifo data */
#define D64_FA_SEL_RFP          0xd0000 /* receive fifo pointers */
#define D64_FA_SEL_RSD          0xe0000 /* receive frame status data */
#define D64_FA_SEL_RSP          0xf0000 /* receive frame status pointers */

/* descriptor control flags 1 */
#define D64_CTRL_COREFLAGS      0x0ff00000      /* core specific flags */
#define D64_CTRL1_EOT           ((u32)1 << 28)  /* end of descriptor table */
#define D64_CTRL1_IOC           ((u32)1 << 29)  /* interrupt on completion */
#define D64_CTRL1_EOF           ((u32)1 << 30)  /* end of frame */
#define D64_CTRL1_SOF           ((u32)1 << 31)  /* start of frame */

/* descriptor control flags 2 */
/* buffer byte count. real data len must <= 16KB */
#define D64_CTRL2_BC_MASK       0x00007fff
/* address extension bits */
#define D64_CTRL2_AE            0x00030000
#define D64_CTRL2_AE_SHIFT      16
/* parity bit */
#define D64_CTRL2_PARITY        0x00040000

/* control flags in the range [27:20] are core-specific and not defined here */
#define D64_CTRL_CORE_MASK      0x0ff00000

#define D64_RX_FRM_STS_LEN      0x0000ffff      /* frame length mask */
#define D64_RX_FRM_STS_OVFL     0x00800000      /* RxOverFlow */
#define D64_RX_FRM_STS_DSCRCNT  0x0f000000  /* no. of descriptors used - 1 */
#define D64_RX_FRM_STS_DATATYPE 0xf0000000      /* core-dependent data type */

/*
 * packet headroom necessary to accommodate the largest header
 * in the system, (i.e TXOFF). By doing, we avoid the need to
 * allocate an extra buffer for the header when bridging to WL.
 * There is a compile time check in wlc.c which ensure that this
 * value is at least as big as TXOFF. This value is used in
 * dma_rxfill().
 */

#define BCMEXTRAHDROOM 172

#define MAXNAMEL        8       /* 8 char names */

/* macros to convert between byte offsets and indexes */
#define B2I(bytes, type)        ((bytes) / sizeof(type))
#define I2B(index, type)        ((index) * sizeof(type))

#define PCI32ADDR_HIGH          0xc0000000      /* address[31:30] */
#define PCI32ADDR_HIGH_SHIFT    30      /* address[31:30] */

#define PCI64ADDR_HIGH          0x80000000      /* address[63] */
#define PCI64ADDR_HIGH_SHIFT    31      /* address[63] */

/*
 * DMA Descriptor
 * Descriptors are only read by the hardware, never written back.
 */
struct dma64desc {
        __le32 ctrl1;   /* misc control bits & bufcount */
        __le32 ctrl2;   /* buffer count and address extension */
        __le32 addrlow; /* memory address of the date buffer, bits 31:0 */
        __le32 addrhigh; /* memory address of the date buffer, bits 63:32 */
};

/* dma engine software state */
struct dma_info {
        struct dma_pub dma; /* exported structure */
        char name[MAXNAMEL];    /* callers name for diag msgs */

        struct bcma_device *core;
        struct device *dmadev;

        /* session information for AMPDU */
        struct brcms_ampdu_session ampdu_session;

        bool dma64;     /* this dma engine is operating in 64-bit mode */
        bool addrext;   /* this dma engine supports DmaExtendedAddrChanges */

        /* 64-bit dma tx engine registers */
        uint d64txregbase;
        /* 64-bit dma rx engine registers */
        uint d64rxregbase;
        /* pointer to dma64 tx descriptor ring */
        struct dma64desc *txd64;
        /* pointer to dma64 rx descriptor ring */
        struct dma64desc *rxd64;

        u16 dmadesc_align;      /* alignment requirement for dma descriptors */

        u16 ntxd;               /* # tx descriptors tunable */
        u16 txin;               /* index of next descriptor to reclaim */
        u16 txout;              /* index of next descriptor to post */
        /* pointer to parallel array of pointers to packets */
        struct sk_buff **txp;
        /* Aligned physical address of descriptor ring */
        dma_addr_t txdpa;
        /* Original physical address of descriptor ring */
        dma_addr_t txdpaorig;
        u16 txdalign;   /* #bytes added to alloc'd mem to align txd */
        u32 txdalloc;   /* #bytes allocated for the ring */
        u32 xmtptrbase; /* When using unaligned descriptors, the ptr register
                         * is not just an index, it needs all 13 bits to be
                         * an offset from the addr register.
                         */

        u16 nrxd;       /* # rx descriptors tunable */
        u16 rxin;       /* index of next descriptor to reclaim */
        u16 rxout;      /* index of next descriptor to post */
        /* pointer to parallel array of pointers to packets */
        struct sk_buff **rxp;
        /* Aligned physical address of descriptor ring */
        dma_addr_t rxdpa;
        /* Original physical address of descriptor ring */
        dma_addr_t rxdpaorig;
        u16 rxdalign;   /* #bytes added to alloc'd mem to align rxd */
        u32 rxdalloc;   /* #bytes allocated for the ring */
        u32 rcvptrbase; /* Base for ptr reg when using unaligned descriptors */

        /* tunables */
        unsigned int rxbufsize; /* rx buffer size in bytes, not including
                                 * the extra headroom
                                 */
        uint rxextrahdrroom;    /* extra rx headroom, reverseved to assist upper
                                 * stack, e.g. some rx pkt buffers will be
                                 * bridged to tx side without byte copying.
                                 * The extra headroom needs to be large enough
                                 * to fit txheader needs. Some dongle driver may
                                 * not need it.
                                 */
        uint nrxpost;           /* # rx buffers to keep posted */
        unsigned int rxoffset;  /* rxcontrol offset */
        /* add to get dma address of descriptor ring, low 32 bits */
        uint ddoffsetlow;
        /*   high 32 bits */
        uint ddoffsethigh;
        /* add to get dma address of data buffer, low 32 bits */
        uint dataoffsetlow;
        /*   high 32 bits */
        uint dataoffsethigh;
        /* descriptor base need to be aligned or not */
        bool aligndesc_4k;
};

/* Check for odd number of 1's */
static u32 parity32(__le32 data)
{
        /* no swap needed for counting 1's */
        u32 par_data = *(u32 *)&data;

        par_data ^= par_data >> 16;
        par_data ^= par_data >> 8;
        par_data ^= par_data >> 4;
        par_data ^= par_data >> 2;
        par_data ^= par_data >> 1;

        return par_data & 1;
}

static bool dma64_dd_parity(struct dma64desc *dd)
{
        return parity32(dd->addrlow ^ dd->addrhigh ^ dd->ctrl1 ^ dd->ctrl2);
}

/* descriptor bumping functions */

static uint xxd(uint x, uint n)
{
        return x & (n - 1); /* faster than %, but n must be power of 2 */
}

static uint txd(struct dma_info *di, uint x)
{
        return xxd(x, di->ntxd);
}

static uint rxd(struct dma_info *di, uint x)
{
        return xxd(x, di->nrxd);
}

static uint nexttxd(struct dma_info *di, uint i)
{
        return txd(di, i + 1);
}

static uint prevtxd(struct dma_info *di, uint i)
{
        return txd(di, i - 1);
}

static uint nextrxd(struct dma_info *di, uint i)
{
        return rxd(di, i + 1);
}

static uint ntxdactive(struct dma_info *di, uint h, uint t)
{
        return txd(di, t-h);
}

static uint nrxdactive(struct dma_info *di, uint h, uint t)
{
        return rxd(di, t-h);
}

static uint _dma_ctrlflags(struct dma_info *di, uint mask, uint flags)
{
        uint dmactrlflags;

        if (di == NULL)
                return 0;

        dmactrlflags = di->dma.dmactrlflags;
        dmactrlflags &= ~mask;
        dmactrlflags |= flags;

        /* If trying to enable parity, check if parity is actually supported */
        if (dmactrlflags & DMA_CTRL_PEN) {
                u32 control;

                control = bcma_read32(di->core, DMA64TXREGOFFS(di, control));
                bcma_write32(di->core, DMA64TXREGOFFS(di, control),
                      control | D64_XC_PD);
                if (bcma_read32(di->core, DMA64TXREGOFFS(di, control)) &
                    D64_XC_PD)
                        /* We *can* disable it so it is supported,
                         * restore control register
                         */
                        bcma_write32(di->core, DMA64TXREGOFFS(di, control),
                                     control);
                else
                        /* Not supported, don't allow it to be enabled */
                        dmactrlflags &= ~DMA_CTRL_PEN;
        }

        di->dma.dmactrlflags = dmactrlflags;

        return dmactrlflags;
}

static bool _dma64_addrext(struct dma_info *di, uint ctrl_offset)
{
        u32 w;
        bcma_set32(di->core, ctrl_offset, D64_XC_AE);
        w = bcma_read32(di->core, ctrl_offset);
        bcma_mask32(di->core, ctrl_offset, ~D64_XC_AE);
        return (w & D64_XC_AE) == D64_XC_AE;
}

/*
 * return true if this dma engine supports DmaExtendedAddrChanges,
 * otherwise false
 */
static bool _dma_isaddrext(struct dma_info *di)
{
        /* DMA64 supports full 32- or 64-bit operation. AE is always valid */

        /* not all tx or rx channel are available */
        if (di->d64txregbase != 0) {
                if (!_dma64_addrext(di, DMA64TXREGOFFS(di, control)))
                        brcms_dbg_dma(di->core,
                                      "%s: DMA64 tx doesn't have AE set\n",
                                      di->name);
                return true;
        } else if (di->d64rxregbase != 0) {
                if (!_dma64_addrext(di, DMA64RXREGOFFS(di, control)))
                        brcms_dbg_dma(di->core,
                                      "%s: DMA64 rx doesn't have AE set\n",
                                      di->name);
                return true;
        }

        return false;
}

static bool _dma_descriptor_align(struct dma_info *di)
{
        u32 addrl;

        /* Check to see if the descriptors need to be aligned on 4K/8K or not */
        if (di->d64txregbase != 0) {
                bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow), 0xff0);
                addrl = bcma_read32(di->core, DMA64TXREGOFFS(di, addrlow));
                if (addrl != 0)
                        return false;
        } else if (di->d64rxregbase != 0) {
                bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow), 0xff0);
                addrl = bcma_read32(di->core, DMA64RXREGOFFS(di, addrlow));
                if (addrl != 0)
                        return false;
        }
        return true;
}

/*
 * Descriptor table must start at the DMA hardware dictated alignment, so
 * allocated memory must be large enough to support this requirement.
 */
static void *dma_alloc_consistent(struct dma_info *di, uint size,
                                  u16 align_bits, uint *alloced,
                                  dma_addr_t *pap)
{
        if (align_bits) {
                u16 align = (1 << align_bits);
                if (!IS_ALIGNED(PAGE_SIZE, align))
                        size += align;
                *alloced = size;
        }
        return dma_alloc_coherent(di->dmadev, size, pap, GFP_ATOMIC);
}

static
u8 dma_align_sizetobits(uint size)
{
        u8 bitpos = 0;
        while (size >>= 1)
                bitpos++;
        return bitpos;
}

/* This function ensures that the DMA descriptor ring will not get allocated
 * across Page boundary. If the allocation is done across the page boundary
 * at the first time, then it is freed and the allocation is done at
 * descriptor ring size aligned location. This will ensure that the ring will
 * not cross page boundary
 */
static void *dma_ringalloc(struct dma_info *di, u32 boundary, uint size,
                           u16 *alignbits, uint *alloced,
                           dma_addr_t *descpa)
{
        void *va;
        u32 desc_strtaddr;
        u32 alignbytes = 1 << *alignbits;

        va = dma_alloc_consistent(di, size, *alignbits, alloced, descpa);

        if (NULL == va)
                return NULL;

        desc_strtaddr = (u32) roundup((unsigned long)va, alignbytes);
        if (((desc_strtaddr + size - 1) & boundary) != (desc_strtaddr
                                                        & boundary)) {
                *alignbits = dma_align_sizetobits(size);
                dma_free_coherent(di->dmadev, *alloced, va, *descpa);
                va = dma_alloc_consistent(di, size, *alignbits,
                        alloced, descpa);
        }
        return va;
}

static bool dma64_alloc(struct dma_info *di, uint direction)
{
        u16 size;
        uint ddlen;
        void *va;
        uint alloced = 0;
        u16 align;
        u16 align_bits;

        ddlen = sizeof(struct dma64desc);

        size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
        align_bits = di->dmadesc_align;
        align = (1 << align_bits);

        if (direction == DMA_TX) {
                va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
                        &alloced, &di->txdpaorig);
                if (va == NULL) {
                        brcms_dbg_dma(di->core,
                                      "%s: DMA_ALLOC_CONSISTENT(ntxd) failed\n",
                                      di->name);
                        return false;
                }
                align = (1 << align_bits);
                di->txd64 = (struct dma64desc *)
                                        roundup((unsigned long)va, align);
                di->txdalign = (uint) ((s8 *)di->txd64 - (s8 *) va);
                di->txdpa = di->txdpaorig + di->txdalign;
                di->txdalloc = alloced;
        } else {
                va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
                        &alloced, &di->rxdpaorig);
                if (va == NULL) {
                        brcms_dbg_dma(di->core,
                                      "%s: DMA_ALLOC_CONSISTENT(nrxd) failed\n",
                                      di->name);
                        return false;
                }
                align = (1 << align_bits);
                di->rxd64 = (struct dma64desc *)
                                        roundup((unsigned long)va, align);
                di->rxdalign = (uint) ((s8 *)di->rxd64 - (s8 *) va);
                di->rxdpa = di->rxdpaorig + di->rxdalign;
                di->rxdalloc = alloced;
        }

        return true;
}

static bool _dma_alloc(struct dma_info *di, uint direction)
{
        return dma64_alloc(di, direction);
}

struct dma_pub *dma_attach(char *name, struct brcms_c_info *wlc,
                           uint txregbase, uint rxregbase, uint ntxd, uint nrxd,
                           uint rxbufsize, int rxextheadroom,
                           uint nrxpost, uint rxoffset)
{
        struct si_pub *sih = wlc->hw->sih;
        struct bcma_device *core = wlc->hw->d11core;
        struct dma_info *di;
        u8 rev = core->id.rev;
        uint size;
        struct si_info *sii = container_of(sih, struct si_info, pub);

        /* allocate private info structure */
        di = kzalloc_obj(*di, GFP_ATOMIC);
        if (di == NULL)
                return NULL;

        di->dma64 =
                ((bcma_aread32(core, BCMA_IOST) & SISF_DMA64) == SISF_DMA64);

        /* init dma reg info */
        di->core = core;
        di->d64txregbase = txregbase;
        di->d64rxregbase = rxregbase;

        /*
         * Default flags (which can be changed by the driver calling
         * dma_ctrlflags before enable): For backwards compatibility
         * both Rx Overflow Continue and Parity are DISABLED.
         */
        _dma_ctrlflags(di, DMA_CTRL_ROC | DMA_CTRL_PEN, 0);

        brcms_dbg_dma(di->core, "%s: %s flags 0x%x ntxd %d nrxd %d "
                      "rxbufsize %d rxextheadroom %d nrxpost %d rxoffset %d "
                      "txregbase %u rxregbase %u\n", name, "DMA64",
                      di->dma.dmactrlflags, ntxd, nrxd, rxbufsize,
                      rxextheadroom, nrxpost, rxoffset, txregbase, rxregbase);

        /* make a private copy of our callers name */
        strscpy(di->name, name, sizeof(di->name));

        di->dmadev = core->dma_dev;

        /* save tunables */
        di->ntxd = (u16) ntxd;
        di->nrxd = (u16) nrxd;

        /* the actual dma size doesn't include the extra headroom */
        di->rxextrahdrroom =
            (rxextheadroom == -1) ? BCMEXTRAHDROOM : rxextheadroom;
        if (rxbufsize > BCMEXTRAHDROOM)
                di->rxbufsize = (u16) (rxbufsize - di->rxextrahdrroom);
        else
                di->rxbufsize = (u16) rxbufsize;

        di->nrxpost = (u16) nrxpost;
        di->rxoffset = (u8) rxoffset;

        /*
         * figure out the DMA physical address offset for dd and data
         *     PCI/PCIE: they map silicon backplace address to zero
         *     based memory, need offset
         *     Other bus: use zero SI_BUS BIGENDIAN kludge: use sdram
         *     swapped region for data buffer, not descriptor
         */
        di->ddoffsetlow = 0;
        di->dataoffsetlow = 0;
        /* for pci bus, add offset */
        if (sii->icbus->hosttype == BCMA_HOSTTYPE_PCI) {
                /* add offset for pcie with DMA64 bus */
                di->ddoffsetlow = 0;
                di->ddoffsethigh = SI_PCIE_DMA_H32;
        }
        di->dataoffsetlow = di->ddoffsetlow;
        di->dataoffsethigh = di->ddoffsethigh;

        /* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */
        if ((core->id.id == BCMA_CORE_SDIO_DEV)
            && ((rev > 0) && (rev <= 2)))
                di->addrext = false;
        else if ((core->id.id == BCMA_CORE_I2S) &&
                 ((rev == 0) || (rev == 1)))
                di->addrext = false;
        else
                di->addrext = _dma_isaddrext(di);

        /* does the descriptor need to be aligned and if yes, on 4K/8K or not */
        di->aligndesc_4k = _dma_descriptor_align(di);
        if (di->aligndesc_4k) {
                di->dmadesc_align = D64RINGALIGN_BITS;
                if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2))
                        /* for smaller dd table, HW relax alignment reqmnt */
                        di->dmadesc_align = D64RINGALIGN_BITS - 1;
        } else {
                di->dmadesc_align = 4;  /* 16 byte alignment */
        }

        brcms_dbg_dma(di->core, "DMA descriptor align_needed %d, align %d\n",
                      di->aligndesc_4k, di->dmadesc_align);

        /* allocate tx packet pointer vector */
        if (ntxd) {
                size = ntxd * sizeof(void *);
                di->txp = kzalloc(size, GFP_ATOMIC);
                if (di->txp == NULL)
                        goto fail;
        }

        /* allocate rx packet pointer vector */
        if (nrxd) {
                size = nrxd * sizeof(void *);
                di->rxp = kzalloc(size, GFP_ATOMIC);
                if (di->rxp == NULL)
                        goto fail;
        }

        /*
         * allocate transmit descriptor ring, only need ntxd descriptors
         * but it must be aligned
         */
        if (ntxd) {
                if (!_dma_alloc(di, DMA_TX))
                        goto fail;
        }

        /*
         * allocate receive descriptor ring, only need nrxd descriptors
         * but it must be aligned
         */
        if (nrxd) {
                if (!_dma_alloc(di, DMA_RX))
                        goto fail;
        }

        if ((di->ddoffsetlow != 0) && !di->addrext) {
                if (di->txdpa > SI_PCI_DMA_SZ) {
                        brcms_dbg_dma(di->core,
                                      "%s: txdpa 0x%x: addrext not supported\n",
                                      di->name, (u32)di->txdpa);
                        goto fail;
                }
                if (di->rxdpa > SI_PCI_DMA_SZ) {
                        brcms_dbg_dma(di->core,
                                      "%s: rxdpa 0x%x: addrext not supported\n",
                                      di->name, (u32)di->rxdpa);
                        goto fail;
                }
        }

        /* Initialize AMPDU session */
        brcms_c_ampdu_reset_session(&di->ampdu_session, wlc);

        brcms_dbg_dma(di->core,
                      "ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh 0x%x addrext %d\n",
                      di->ddoffsetlow, di->ddoffsethigh,
                      di->dataoffsetlow, di->dataoffsethigh,
                      di->addrext);

        return (struct dma_pub *) di;

 fail:
        dma_detach((struct dma_pub *)di);
        return NULL;
}

static inline void
dma64_dd_upd(struct dma_info *di, struct dma64desc *ddring,
             dma_addr_t pa, uint outidx, u32 *flags, u32 bufcount)
{
        u32 ctrl2 = bufcount & D64_CTRL2_BC_MASK;

        /* PCI bus with big(>1G) physical address, use address extension */
        if ((di->dataoffsetlow == 0) || !(pa & PCI32ADDR_HIGH)) {
                ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);
                ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);
                ddring[outidx].ctrl1 = cpu_to_le32(*flags);
                ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);
        } else {
                /* address extension for 32-bit PCI */
                u32 ae;

                ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
                pa &= ~PCI32ADDR_HIGH;

                ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE;
                ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);
                ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);
                ddring[outidx].ctrl1 = cpu_to_le32(*flags);
                ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);
        }
        if (di->dma.dmactrlflags & DMA_CTRL_PEN) {
                if (dma64_dd_parity(&ddring[outidx]))
                        ddring[outidx].ctrl2 =
                             cpu_to_le32(ctrl2 | D64_CTRL2_PARITY);
        }
}

/* !! may be called with core in reset */
void dma_detach(struct dma_pub *pub)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);

        brcms_dbg_dma(di->core, "%s:\n", di->name);

        /* free dma descriptor rings */
        if (di->txd64)
                dma_free_coherent(di->dmadev, di->txdalloc,
                                  ((s8 *)di->txd64 - di->txdalign),
                                  (di->txdpaorig));
        if (di->rxd64)
                dma_free_coherent(di->dmadev, di->rxdalloc,
                                  ((s8 *)di->rxd64 - di->rxdalign),
                                  (di->rxdpaorig));

        /* free packet pointer vectors */
        kfree(di->txp);
        kfree(di->rxp);

        /* free our private info structure */
        kfree(di);

}

/* initialize descriptor table base address */
static void
_dma_ddtable_init(struct dma_info *di, uint direction, dma_addr_t pa)
{
        if (!di->aligndesc_4k) {
                if (direction == DMA_TX)
                        di->xmtptrbase = pa;
                else
                        di->rcvptrbase = pa;
        }

        if ((di->ddoffsetlow == 0)
            || !(pa & PCI32ADDR_HIGH)) {
                if (direction == DMA_TX) {
                        bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),
                                     pa + di->ddoffsetlow);
                        bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),
                                     di->ddoffsethigh);
                } else {
                        bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),
                                     pa + di->ddoffsetlow);
                        bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),
                                     di->ddoffsethigh);
                }
        } else {
                /* DMA64 32bits address extension */
                u32 ae;

                /* shift the high bit(s) from pa to ae */
                ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
                pa &= ~PCI32ADDR_HIGH;

                if (direction == DMA_TX) {
                        bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),
                                     pa + di->ddoffsetlow);
                        bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),
                                     di->ddoffsethigh);
                        bcma_maskset32(di->core, DMA64TXREGOFFS(di, control),
                                       D64_XC_AE, (ae << D64_XC_AE_SHIFT));
                } else {
                        bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),
                                     pa + di->ddoffsetlow);
                        bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),
                                     di->ddoffsethigh);
                        bcma_maskset32(di->core, DMA64RXREGOFFS(di, control),
                                       D64_RC_AE, (ae << D64_RC_AE_SHIFT));
                }
        }
}

static void _dma_rxenable(struct dma_info *di)
{
        uint dmactrlflags = di->dma.dmactrlflags;
        u32 control;

        brcms_dbg_dma(di->core, "%s:\n", di->name);

        control = D64_RC_RE | (bcma_read32(di->core,
                                           DMA64RXREGOFFS(di, control)) &
                               D64_RC_AE);

        if ((dmactrlflags & DMA_CTRL_PEN) == 0)
                control |= D64_RC_PD;

        if (dmactrlflags & DMA_CTRL_ROC)
                control |= D64_RC_OC;

        bcma_write32(di->core, DMA64RXREGOFFS(di, control),
                ((di->rxoffset << D64_RC_RO_SHIFT) | control));
}

void dma_rxinit(struct dma_pub *pub)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);

        brcms_dbg_dma(di->core, "%s:\n", di->name);

        if (di->nrxd == 0)
                return;

        di->rxin = di->rxout = 0;

        /* clear rx descriptor ring */
        memset(di->rxd64, '\0', di->nrxd * sizeof(struct dma64desc));

        /* DMA engine with out alignment requirement requires table to be inited
         * before enabling the engine
         */
        if (!di->aligndesc_4k)
                _dma_ddtable_init(di, DMA_RX, di->rxdpa);

        _dma_rxenable(di);

        if (di->aligndesc_4k)
                _dma_ddtable_init(di, DMA_RX, di->rxdpa);
}

static struct sk_buff *dma64_getnextrxp(struct dma_info *di, bool forceall)
{
        uint i, curr;
        struct sk_buff *rxp;
        dma_addr_t pa;

        i = di->rxin;

        /* return if no packets posted */
        if (i == di->rxout)
                return NULL;

        curr =
            B2I(((bcma_read32(di->core,
                              DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) -
                 di->rcvptrbase) & D64_RS0_CD_MASK, struct dma64desc);

        /* ignore curr if forceall */
        if (!forceall && (i == curr))
                return NULL;

        /* get the packet pointer that corresponds to the rx descriptor */
        rxp = di->rxp[i];
        di->rxp[i] = NULL;

        pa = le32_to_cpu(di->rxd64[i].addrlow) - di->dataoffsetlow;

        /* clear this packet from the descriptor ring */
        dma_unmap_single(di->dmadev, pa, di->rxbufsize, DMA_FROM_DEVICE);

        di->rxd64[i].addrlow = cpu_to_le32(0xdeadbeef);
        di->rxd64[i].addrhigh = cpu_to_le32(0xdeadbeef);

        di->rxin = nextrxd(di, i);

        return rxp;
}

static struct sk_buff *_dma_getnextrxp(struct dma_info *di, bool forceall)
{
        if (di->nrxd == 0)
                return NULL;

        return dma64_getnextrxp(di, forceall);
}

/*
 * !! rx entry routine
 * returns the number packages in the next frame, or 0 if there are no more
 *   if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is
 *   supported with pkts chain
 *   otherwise, it's treated as giant pkt and will be tossed.
 *   The DMA scattering starts with normal DMA header, followed by first
 *   buffer data. After it reaches the max size of buffer, the data continues
 *   in next DMA descriptor buffer WITHOUT DMA header
 */
int dma_rx(struct dma_pub *pub, struct sk_buff_head *skb_list)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);
        struct sk_buff_head dma_frames;
        struct sk_buff *p, *next;
        uint len;
        uint pkt_len;
        int resid = 0;
        int pktcnt = 1;

        skb_queue_head_init(&dma_frames);
 next_frame:
        p = _dma_getnextrxp(di, false);
        if (p == NULL)
                return 0;

        len = le16_to_cpu(*(__le16 *) (p->data));
        brcms_dbg_dma(di->core, "%s: dma_rx len %d\n", di->name, len);
        dma_spin_for_len(len, p);

        /* set actual length */
        pkt_len = min((di->rxoffset + len), di->rxbufsize);
        __skb_trim(p, pkt_len);
        skb_queue_tail(&dma_frames, p);
        resid = len - (di->rxbufsize - di->rxoffset);

        /* check for single or multi-buffer rx */
        if (resid > 0) {
                while ((resid > 0) && (p = _dma_getnextrxp(di, false))) {
                        pkt_len = min_t(uint, resid, di->rxbufsize);
                        __skb_trim(p, pkt_len);
                        skb_queue_tail(&dma_frames, p);
                        resid -= di->rxbufsize;
                        pktcnt++;
                }

#ifdef DEBUG
                if (resid > 0) {
                        uint cur;
                        cur =
                            B2I(((bcma_read32(di->core,
                                              DMA64RXREGOFFS(di, status0)) &
                                  D64_RS0_CD_MASK) - di->rcvptrbase) &
                                D64_RS0_CD_MASK, struct dma64desc);
                        brcms_dbg_dma(di->core,
                                      "rxin %d rxout %d, hw_curr %d\n",
                                      di->rxin, di->rxout, cur);
                }
#endif                          /* DEBUG */

                if ((di->dma.dmactrlflags & DMA_CTRL_RXMULTI) == 0) {
                        brcms_dbg_dma(di->core, "%s: bad frame length (%d)\n",
                                      di->name, len);
                        skb_queue_walk_safe(&dma_frames, p, next) {
                                skb_unlink(p, &dma_frames);
                                brcmu_pkt_buf_free_skb(p);
                        }
                        di->dma.rxgiants++;
                        pktcnt = 1;
                        goto next_frame;
                }
        }

        skb_queue_splice_tail(&dma_frames, skb_list);
        return pktcnt;
}

static bool dma64_rxidle(struct dma_info *di)
{
        brcms_dbg_dma(di->core, "%s:\n", di->name);

        if (di->nrxd == 0)
                return true;

        return ((bcma_read32(di->core,
                             DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) ==
                (bcma_read32(di->core, DMA64RXREGOFFS(di, ptr)) &
                 D64_RS0_CD_MASK));
}

static bool dma64_txidle(struct dma_info *di)
{
        if (di->ntxd == 0)
                return true;

        return ((bcma_read32(di->core,
                             DMA64TXREGOFFS(di, status0)) & D64_XS0_CD_MASK) ==
                (bcma_read32(di->core, DMA64TXREGOFFS(di, ptr)) &
                 D64_XS0_CD_MASK));
}

/*
 * post receive buffers
 *  Return false if refill failed completely or dma mapping failed. The ring
 *  is empty, which will stall the rx dma and user might want to call rxfill
 *  again asap. This is unlikely to happen on a memory-rich NIC, but often on
 *  memory-constrained dongle.
 */
bool dma_rxfill(struct dma_pub *pub)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);
        struct sk_buff *p;
        u16 rxin, rxout;
        u32 flags = 0;
        uint n;
        uint i;
        dma_addr_t pa;
        uint extra_offset = 0;
        bool ring_empty;

        ring_empty = false;

        /*
         * Determine how many receive buffers we're lacking
         * from the full complement, allocate, initialize,
         * and post them, then update the chip rx lastdscr.
         */

        rxin = di->rxin;
        rxout = di->rxout;

        n = di->nrxpost - nrxdactive(di, rxin, rxout);

        brcms_dbg_dma(di->core, "%s: post %d\n", di->name, n);

        if (di->rxbufsize > BCMEXTRAHDROOM)
                extra_offset = di->rxextrahdrroom;

        for (i = 0; i < n; i++) {
                /*
                 * the di->rxbufsize doesn't include the extra headroom,
                 * we need to add it to the size to be allocated
                 */
                p = brcmu_pkt_buf_get_skb(di->rxbufsize + extra_offset);

                if (p == NULL) {
                        brcms_dbg_dma(di->core, "%s: out of rxbufs\n",
                                      di->name);
                        if (i == 0 && dma64_rxidle(di)) {
                                brcms_dbg_dma(di->core, "%s: ring is empty !\n",
                                              di->name);
                                ring_empty = true;
                        }
                        di->dma.rxnobuf++;
                        break;
                }
                /* reserve an extra headroom, if applicable */
                if (extra_offset)
                        skb_pull(p, extra_offset);

                /* Do a cached write instead of uncached write since DMA_MAP
                 * will flush the cache.
                 */
                *(u32 *) (p->data) = 0;

                pa = dma_map_single(di->dmadev, p->data, di->rxbufsize,
                                    DMA_FROM_DEVICE);
                if (dma_mapping_error(di->dmadev, pa)) {
                        brcmu_pkt_buf_free_skb(p);
                        return false;
                }

                /* save the free packet pointer */
                di->rxp[rxout] = p;

                /* reset flags for each descriptor */
                flags = 0;
                if (rxout == (di->nrxd - 1))
                        flags = D64_CTRL1_EOT;

                dma64_dd_upd(di, di->rxd64, pa, rxout, &flags,
                             di->rxbufsize);
                rxout = nextrxd(di, rxout);
        }

        di->rxout = rxout;

        /* update the chip lastdscr pointer */
        bcma_write32(di->core, DMA64RXREGOFFS(di, ptr),
              di->rcvptrbase + I2B(rxout, struct dma64desc));

        return ring_empty;
}

void dma_rxreclaim(struct dma_pub *pub)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);
        struct sk_buff *p;

        brcms_dbg_dma(di->core, "%s:\n", di->name);

        while ((p = _dma_getnextrxp(di, true)))
                brcmu_pkt_buf_free_skb(p);
}

void dma_counterreset(struct dma_pub *pub)
{
        /* reset all software counters */
        pub->rxgiants = 0;
        pub->rxnobuf = 0;
        pub->txnobuf = 0;
}

/* get the address of the var in order to change later */
unsigned long dma_getvar(struct dma_pub *pub, const char *name)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);

        if (!strcmp(name, "&txavail"))
                return (unsigned long)&(di->dma.txavail);
        return 0;
}

/* 64-bit DMA functions */

void dma_txinit(struct dma_pub *pub)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);
        u32 control = D64_XC_XE;

        brcms_dbg_dma(di->core, "%s:\n", di->name);

        if (di->ntxd == 0)
                return;

        di->txin = di->txout = 0;
        di->dma.txavail = di->ntxd - 1;

        /* clear tx descriptor ring */
        memset(di->txd64, '\0', (di->ntxd * sizeof(struct dma64desc)));

        /* DMA engine with out alignment requirement requires table to be inited
         * before enabling the engine
         */
        if (!di->aligndesc_4k)
                _dma_ddtable_init(di, DMA_TX, di->txdpa);

        if ((di->dma.dmactrlflags & DMA_CTRL_PEN) == 0)
                control |= D64_XC_PD;
        bcma_set32(di->core, DMA64TXREGOFFS(di, control), control);

        /* DMA engine with alignment requirement requires table to be inited
         * before enabling the engine
         */
        if (di->aligndesc_4k)
                _dma_ddtable_init(di, DMA_TX, di->txdpa);
}

void dma_txsuspend(struct dma_pub *pub)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);

        brcms_dbg_dma(di->core, "%s:\n", di->name);

        if (di->ntxd == 0)
                return;

        bcma_set32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);
}

void dma_txresume(struct dma_pub *pub)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);

        brcms_dbg_dma(di->core, "%s:\n", di->name);

        if (di->ntxd == 0)
                return;

        bcma_mask32(di->core, DMA64TXREGOFFS(di, control), ~D64_XC_SE);
}

bool dma_txsuspended(struct dma_pub *pub)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);

        return (di->ntxd == 0) ||
               ((bcma_read32(di->core,
                             DMA64TXREGOFFS(di, control)) & D64_XC_SE) ==
                D64_XC_SE);
}

void dma_txreclaim(struct dma_pub *pub, enum txd_range range)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);
        struct sk_buff *p;

        brcms_dbg_dma(di->core, "%s: %s\n",
                      di->name,
                      range == DMA_RANGE_ALL ? "all" :
                      range == DMA_RANGE_TRANSMITTED ? "transmitted" :
                      "transferred");

        if (di->txin == di->txout)
                return;

        while ((p = dma_getnexttxp(pub, range))) {
                /* For unframed data, we don't have any packets to free */
                if (!(di->dma.dmactrlflags & DMA_CTRL_UNFRAMED))
                        brcmu_pkt_buf_free_skb(p);
        }
}

bool dma_txreset(struct dma_pub *pub)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);
        u32 status;

        if (di->ntxd == 0)
                return true;

        /* suspend tx DMA first */
        bcma_write32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);
        SPINWAIT(((status =
                   (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &
                    D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED) &&
                  (status != D64_XS0_XS_IDLE) && (status != D64_XS0_XS_STOPPED),
                 10000);

        bcma_write32(di->core, DMA64TXREGOFFS(di, control), 0);
        SPINWAIT(((status =
                   (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &
                    D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED), 10000);

        /* wait for the last transaction to complete */
        udelay(300);

        return status == D64_XS0_XS_DISABLED;
}

bool dma_rxreset(struct dma_pub *pub)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);
        u32 status;

        if (di->nrxd == 0)
                return true;

        bcma_write32(di->core, DMA64RXREGOFFS(di, control), 0);
        SPINWAIT(((status =
                   (bcma_read32(di->core, DMA64RXREGOFFS(di, status0)) &
                    D64_RS0_RS_MASK)) != D64_RS0_RS_DISABLED), 10000);

        return status == D64_RS0_RS_DISABLED;
}

static void dma_txenq(struct dma_info *di, struct sk_buff *p)
{
        unsigned char *data;
        uint len;
        u16 txout;
        u32 flags = 0;
        dma_addr_t pa;

        txout = di->txout;

        if (WARN_ON(nexttxd(di, txout) == di->txin))
                return;

        /*
         * obtain and initialize transmit descriptor entry.
         */
        data = p->data;
        len = p->len;

        /* get physical address of buffer start */
        pa = dma_map_single(di->dmadev, data, len, DMA_TO_DEVICE);
        /* if mapping failed, free skb */
        if (dma_mapping_error(di->dmadev, pa)) {
                brcmu_pkt_buf_free_skb(p);
                return;
        }
        /* With a DMA segment list, Descriptor table is filled
         * using the segment list instead of looping over
         * buffers in multi-chain DMA. Therefore, EOF for SGLIST
         * is when end of segment list is reached.
         */
        flags = D64_CTRL1_SOF | D64_CTRL1_IOC | D64_CTRL1_EOF;
        if (txout == (di->ntxd - 1))
                flags |= D64_CTRL1_EOT;

        dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);

        txout = nexttxd(di, txout);

        /* save the packet */
        di->txp[prevtxd(di, txout)] = p;

        /* bump the tx descriptor index */
        di->txout = txout;
}

static void ampdu_finalize(struct dma_info *di)
{
        struct brcms_ampdu_session *session = &di->ampdu_session;
        struct sk_buff *p;

        trace_brcms_ampdu_session(&session->wlc->hw->d11core->dev,
                                  session->max_ampdu_len,
                                  session->max_ampdu_frames,
                                  session->ampdu_len,
                                  skb_queue_len(&session->skb_list),
                                  session->dma_len);

        if (WARN_ON(skb_queue_empty(&session->skb_list)))
                return;

        brcms_c_ampdu_finalize(session);

        while (!skb_queue_empty(&session->skb_list)) {
                p = skb_dequeue(&session->skb_list);
                dma_txenq(di, p);
        }

        bcma_write32(di->core, DMA64TXREGOFFS(di, ptr),
                     di->xmtptrbase + I2B(di->txout, struct dma64desc));
        brcms_c_ampdu_reset_session(session, session->wlc);
}

static void prep_ampdu_frame(struct dma_info *di, struct sk_buff *p)
{
        struct brcms_ampdu_session *session = &di->ampdu_session;
        int ret;

        ret = brcms_c_ampdu_add_frame(session, p);
        if (ret == -ENOSPC) {
                /*
                 * AMPDU cannot accommodate this frame. Close out the in-
                 * progress AMPDU session and start a new one.
                 */
                ampdu_finalize(di);
                ret = brcms_c_ampdu_add_frame(session, p);
        }

        WARN_ON(ret);
}

/* Update count of available tx descriptors based on current DMA state */
static void dma_update_txavail(struct dma_info *di)
{
        /*
         * Available space is number of descriptors less the number of
         * active descriptors and the number of queued AMPDU frames.
         */
        di->dma.txavail = di->ntxd - ntxdactive(di, di->txin, di->txout) -
                          skb_queue_len(&di->ampdu_session.skb_list) - 1;
}

/*
 * !! tx entry routine
 * WARNING: call must check the return value for error.
 *   the error(toss frames) could be fatal and cause many subsequent hard
 *   to debug problems
 */
int dma_txfast(struct brcms_c_info *wlc, struct dma_pub *pub,
               struct sk_buff *p)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);
        struct brcms_ampdu_session *session = &di->ampdu_session;
        struct ieee80211_tx_info *tx_info;
        bool is_ampdu;

        /* no use to transmit a zero length packet */
        if (p->len == 0)
                return 0;

        /* return nonzero if out of tx descriptors */
        if (di->dma.txavail == 0 || nexttxd(di, di->txout) == di->txin)
                goto outoftxd;

        tx_info = IEEE80211_SKB_CB(p);
        is_ampdu = tx_info->flags & IEEE80211_TX_CTL_AMPDU;
        if (is_ampdu)
                prep_ampdu_frame(di, p);
        else
                dma_txenq(di, p);

        /* tx flow control */
        dma_update_txavail(di);

        /* kick the chip */
        if (is_ampdu) {
                /*
                 * Start sending data if we've got a full AMPDU, there's
                 * no more space in the DMA ring, or the ring isn't
                 * currently transmitting.
                 */
                if (skb_queue_len(&session->skb_list) == session->max_ampdu_frames ||
                    di->dma.txavail == 0 || dma64_txidle(di))
                        ampdu_finalize(di);
        } else {
                bcma_write32(di->core, DMA64TXREGOFFS(di, ptr),
                             di->xmtptrbase + I2B(di->txout, struct dma64desc));
        }

        return 0;

 outoftxd:
        brcms_dbg_dma(di->core, "%s: out of txds !!!\n", di->name);
        brcmu_pkt_buf_free_skb(p);
        di->dma.txavail = 0;
        di->dma.txnobuf++;
        return -ENOSPC;
}

int dma_txpending(struct dma_pub *pub)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);
        return ntxdactive(di, di->txin, di->txout);
}

/*
 * If we have an active AMPDU session and are not transmitting,
 * this function will force tx to start.
 */
void dma_kick_tx(struct dma_pub *pub)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);
        struct brcms_ampdu_session *session = &di->ampdu_session;

        if (!skb_queue_empty(&session->skb_list) && dma64_txidle(di))
                ampdu_finalize(di);
}

/*
 * Reclaim next completed txd (txds if using chained buffers) in the range
 * specified and return associated packet.
 * If range is DMA_RANGE_TRANSMITTED, reclaim descriptors that have be
 * transmitted as noted by the hardware "CurrDescr" pointer.
 * If range is DMA_RANGE_TRANSFERED, reclaim descriptors that have be
 * transferred by the DMA as noted by the hardware "ActiveDescr" pointer.
 * If range is DMA_RANGE_ALL, reclaim all txd(s) posted to the ring and
 * return associated packet regardless of the value of hardware pointers.
 */
struct sk_buff *dma_getnexttxp(struct dma_pub *pub, enum txd_range range)
{
        struct dma_info *di = container_of(pub, struct dma_info, dma);
        u16 start, end, i;
        u16 active_desc;
        struct sk_buff *txp;

        brcms_dbg_dma(di->core, "%s: %s\n",
                      di->name,
                      range == DMA_RANGE_ALL ? "all" :
                      range == DMA_RANGE_TRANSMITTED ? "transmitted" :
                      "transferred");

        if (di->ntxd == 0)
                return NULL;

        txp = NULL;

        start = di->txin;
        if (range == DMA_RANGE_ALL)
                end = di->txout;
        else {
                end = (u16) (B2I(((bcma_read32(di->core,
                                               DMA64TXREGOFFS(di, status0)) &
                                   D64_XS0_CD_MASK) - di->xmtptrbase) &
                                 D64_XS0_CD_MASK, struct dma64desc));

                if (range == DMA_RANGE_TRANSFERED) {
                        active_desc =
                                (u16)(bcma_read32(di->core,
                                                  DMA64TXREGOFFS(di, status1)) &
                                      D64_XS1_AD_MASK);
                        active_desc =
                            (active_desc - di->xmtptrbase) & D64_XS0_CD_MASK;
                        active_desc = B2I(active_desc, struct dma64desc);
                        if (end != active_desc)
                                end = prevtxd(di, active_desc);
                }
        }

        if ((start == 0) && (end > di->txout))
                goto bogus;

        for (i = start; i != end && !txp; i = nexttxd(di, i)) {
                dma_addr_t pa;
                uint size;

                pa = le32_to_cpu(di->txd64[i].addrlow) - di->dataoffsetlow;

                size =
                    (le32_to_cpu(di->txd64[i].ctrl2) &
                     D64_CTRL2_BC_MASK);

                di->txd64[i].addrlow = cpu_to_le32(0xdeadbeef);
                di->txd64[i].addrhigh = cpu_to_le32(0xdeadbeef);

                txp = di->txp[i];
                di->txp[i] = NULL;

                dma_unmap_single(di->dmadev, pa, size, DMA_TO_DEVICE);
        }

        di->txin = i;

        /* tx flow control */
        dma_update_txavail(di);

        return txp;

 bogus:
        brcms_dbg_dma(di->core, "bogus curr: start %d end %d txout %d\n",
                      start, end, di->txout);
        return NULL;
}

/*
 * Mac80211 initiated actions sometimes require packets in the DMA queue to be
 * modified. The modified portion of the packet is not under control of the DMA
 * engine. This function calls a caller-supplied function for each packet in
 * the caller specified dma chain.
 */
void dma_walk_packets(struct dma_pub *dmah, void (*callback_fnc)
                      (void *pkt, void *arg_a), void *arg_a)
{
        struct dma_info *di = container_of(dmah, struct dma_info, dma);
        uint i =   di->txin;
        uint end = di->txout;
        struct sk_buff *skb;
        struct ieee80211_tx_info *tx_info;

        while (i != end) {
                skb = di->txp[i];
                if (skb != NULL) {
                        tx_info = (struct ieee80211_tx_info *)skb->cb;
                        (callback_fnc)(tx_info, arg_a);
                }
                i = nexttxd(di, i);
        }
}