/*      $NetBSD: gtidmac.c,v 1.21 2024/06/12 20:03:56 andvar Exp $      */
/*
 * Copyright (c) 2008, 2012, 2016 KIYOHARA Takashi
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: gtidmac.c,v 1.21 2024/06/12 20:03:56 andvar Exp $");

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/device.h>
#include <sys/errno.h>
#include <sys/endian.h>
#include <sys/kmem.h>

#include <uvm/uvm_param.h>      /* For PAGE_SIZE */

#include <dev/dmover/dmovervar.h>

#include <dev/marvell/gtidmacreg.h>
#include <dev/marvell/gtidmacvar.h>
#include <dev/marvell/marvellreg.h>
#include <dev/marvell/marvellvar.h>

#include <prop/proplib.h>

#include "locators.h"

#ifdef GTIDMAC_DEBUG
#define DPRINTF(x)      if (gtidmac_debug) printf x
int gtidmac_debug = 0;
#else
#define DPRINTF(x)
#endif

#define GTIDMAC_NDESC           64
#define GTIDMAC_MAXCHAN         8
#define MVXORE_NDESC            128
#define MVXORE_MAXCHAN          2

#define GTIDMAC_NSEGS           ((GTIDMAC_MAXXFER + PAGE_SIZE - 1) / PAGE_SIZE)
#define MVXORE_NSEGS            ((MVXORE_MAXXFER + PAGE_SIZE - 1) / PAGE_SIZE)


struct gtidmac_softc;

struct gtidmac_function {
        int (*chan_alloc)(void *, bus_dmamap_t **, bus_dmamap_t **, void *);
        void (*chan_free)(void *, int);
        int (*dma_setup)(void *, int, int, bus_dmamap_t *, bus_dmamap_t *,
                         bus_size_t);
        void (*dma_start)(void *, int,
                          void (*dma_done_cb)(void *, int, bus_dmamap_t *,
                                                      bus_dmamap_t *, int));
        uint32_t (*dma_finish)(void *, int, int);
};

struct gtidmac_dma_desc {
        int dd_index;
        union {
                struct gtidmac_desc *idmac_vaddr;
                struct mvxore_desc *xore_vaddr;
        } dd_vaddr;
#define dd_idmac_vaddr  dd_vaddr.idmac_vaddr
#define dd_xore_vaddr   dd_vaddr.xore_vaddr
        paddr_t dd_paddr;
        SLIST_ENTRY(gtidmac_dma_desc) dd_next;
};

struct gtidmac_softc {
        device_t sc_dev;

        bus_space_tag_t sc_iot;
        bus_space_handle_t sc_ioh;

        bus_dma_tag_t sc_dmat;
        struct gtidmac_dma_desc *sc_dd_buffer;
        bus_dma_segment_t sc_pattern_segment;
        struct {
                u_char pbuf[16];        /* 16byte/pattern */
        } *sc_pbuf;                     /*   x256 pattern */

        int sc_gtidmac_nchan;
        struct gtidmac_desc *sc_dbuf;
        bus_dmamap_t sc_dmap;
        SLIST_HEAD(, gtidmac_dma_desc) sc_dlist;
        struct {
                bus_dmamap_t chan_in;           /* In dmamap */
                bus_dmamap_t chan_out;          /* Out dmamap */
                uint64_t chan_totalcnt;         /* total transferred byte */
                int chan_ddidx;
                void *chan_running;             /* opaque object data */
                void (*chan_dma_done)(void *, int, bus_dmamap_t *,
                                      bus_dmamap_t *, int);
        } sc_cdesc[GTIDMAC_MAXCHAN];
        struct gtidmac_intr_arg {
                struct gtidmac_softc *ia_sc;
                uint32_t ia_cause;
                uint32_t ia_mask;
                uint32_t ia_eaddr;
                uint32_t ia_eselect;
        } sc_intrarg[GTIDMAC_NINTRRUPT];

        int sc_mvxore_nchan;
        struct mvxore_desc *sc_dbuf_xore;
        bus_dmamap_t sc_dmap_xore;
        SLIST_HEAD(, gtidmac_dma_desc) sc_dlist_xore;
        struct {
                bus_dmamap_t chan_in[MVXORE_NSRC];      /* In dmamap */
                bus_dmamap_t chan_out;                  /* Out dmamap */
                uint64_t chan_totalcnt;                 /* total transferred */
                int chan_ddidx;
                void *chan_running;                     /* opaque object data */
                void (*chan_dma_done)(void *, int, bus_dmamap_t *,
                                      bus_dmamap_t *, int);
        } sc_cdesc_xore[MVXORE_MAXCHAN];

        struct dmover_backend sc_dmb;
        struct dmover_backend sc_dmb_xore;
        int sc_dmb_busy;
};
struct gtidmac_softc *gtidmac_softc = NULL;

static int gtidmac_match(device_t, struct cfdata *, void *);
static void gtidmac_attach(device_t, device_t, void *);

static int gtidmac_intr(void *);
static int mvxore_port0_intr(void *);
static int mvxore_port1_intr(void *);
static int mvxore_intr(struct gtidmac_softc *, int);

static void gtidmac_process(struct dmover_backend *);
static void gtidmac_dmover_run(struct dmover_backend *);
static void gtidmac_dmover_done(void *, int, bus_dmamap_t *, bus_dmamap_t *,
                                int);
static __inline int gtidmac_dmmap_load(struct gtidmac_softc *, bus_dmamap_t,
                                dmover_buffer_type, dmover_buffer *, int);
static __inline void gtidmac_dmmap_unload(struct gtidmac_softc *, bus_dmamap_t, int);

static uint32_t gtidmac_finish(void *, int, int);
static uint32_t mvxore_finish(void *, int, int);

static void gtidmac_wininit(struct gtidmac_softc *, enum marvell_tags *);
static void mvxore_wininit(struct gtidmac_softc *, enum marvell_tags *);

static int gtidmac_buffer_setup(struct gtidmac_softc *);
static int mvxore_buffer_setup(struct gtidmac_softc *);

#ifdef GTIDMAC_DEBUG
static void gtidmac_dump_idmacreg(struct gtidmac_softc *, int);
static void gtidmac_dump_idmacdesc(struct gtidmac_softc *,
                                   struct gtidmac_dma_desc *, uint32_t, int);
static void gtidmac_dump_xorereg(struct gtidmac_softc *, int);
static void gtidmac_dump_xoredesc(struct gtidmac_softc *,
                                  struct gtidmac_dma_desc *, uint32_t, int);
#endif


static struct gtidmac_function gtidmac_functions = {
        .chan_alloc = gtidmac_chan_alloc,
        .chan_free = gtidmac_chan_free,
        .dma_setup = gtidmac_setup,
        .dma_start = gtidmac_start,
        .dma_finish = gtidmac_finish,
};

static struct gtidmac_function mvxore_functions = {
        .chan_alloc = mvxore_chan_alloc,
        .chan_free = mvxore_chan_free,
        .dma_setup = mvxore_setup,
        .dma_start = mvxore_start,
        .dma_finish = mvxore_finish,
};

static const struct dmover_algdesc gtidmac_algdescs[] = {
        {
                .dad_name = DMOVER_FUNC_ZERO,
                .dad_data = &gtidmac_functions,
                .dad_ninputs = 0
        },
        {
                .dad_name = DMOVER_FUNC_FILL8,
                .dad_data = &gtidmac_functions,
                .dad_ninputs = 0
        },
        {
                .dad_name = DMOVER_FUNC_COPY,
                .dad_data = &gtidmac_functions,
                .dad_ninputs = 1
        },
};

static const struct dmover_algdesc mvxore_algdescs[] = {
#if 0
        /*
         * As for these operations, there are a lot of restrictions.  It is
         * necessary to use IDMAC.
         */
        {
                .dad_name = DMOVER_FUNC_ZERO,
                .dad_data = &mvxore_functions,
                .dad_ninputs = 0
        },
        {
                .dad_name = DMOVER_FUNC_FILL8,
                .dad_data = &mvxore_functions,
                .dad_ninputs = 0
        },
#endif
        {
                .dad_name = DMOVER_FUNC_COPY,
                .dad_data = &mvxore_functions,
                .dad_ninputs = 1
        },
        {
                .dad_name = DMOVER_FUNC_ISCSI_CRC32C,
                .dad_data = &mvxore_functions,
                .dad_ninputs = 1
        },
        {
                .dad_name = DMOVER_FUNC_XOR2,
                .dad_data = &mvxore_functions,
                .dad_ninputs = 2
        },
        {
                .dad_name = DMOVER_FUNC_XOR3,
                .dad_data = &mvxore_functions,
                .dad_ninputs = 3
        },
        {
                .dad_name = DMOVER_FUNC_XOR4,
                .dad_data = &mvxore_functions,
                .dad_ninputs = 4
        },
        {
                .dad_name = DMOVER_FUNC_XOR5,
                .dad_data = &mvxore_functions,
                .dad_ninputs = 5
        },
        {
                .dad_name = DMOVER_FUNC_XOR6,
                .dad_data = &mvxore_functions,
                .dad_ninputs = 6
        },
        {
                .dad_name = DMOVER_FUNC_XOR7,
                .dad_data = &mvxore_functions,
                .dad_ninputs = 7
        },
        {
                .dad_name = DMOVER_FUNC_XOR8,
                .dad_data = &mvxore_functions,
                .dad_ninputs = 8
        },
};

static int orion_88f5182_xore_irqs[] = { 30, 31 };
static int kirkwood_xore_irqs[] = { 5, 6, 7, 8 };
static int dove_xore_irqs[] = { 39, 40, 42, 43 };
static int armadaxp_xore_irqs0[] = { 51, 52 };
static int armadaxp_xore_irqs1[] = { 94, 95 };

static struct {
        int model;
        int idmac_nchan;
        int idmac_irq;
        int xore_nchan;
        int *xore_irqs;
} channels[] = {
        /*
         * Marvell System Controllers:
         * need irqs in attach_args.
         */
        { MARVELL_DISCOVERY,            8, -1, 0, NULL },
        { MARVELL_DISCOVERY_II,         8, -1, 0, NULL },
        { MARVELL_DISCOVERY_III,        8, -1, 0, NULL },
#if 0
        { MARVELL_DISCOVERY_LT,         4, -1, 2, NULL },
        { MARVELL_DISCOVERY_V,          4, -1, 2, NULL },
        { MARVELL_DISCOVERY_VI,         4, -1, 2, NULL },               ????
#endif

        /*
         * Marvell System on Chips:
         * No need irqs in attach_args.  We always connecting to interrupt-pin
         * statically.
         */
        { MARVELL_ORION_1_88F1181,      4, 24, 0, NULL },
        { MARVELL_ORION_2_88F1281,      4, 24, 0, NULL },
        { MARVELL_ORION_1_88F5082,      4, 24, 0, NULL },
        { MARVELL_ORION_1_88F5180N,     4, 24, 0, NULL },
        { MARVELL_ORION_1_88F5181,      4, 24, 0, NULL },
        { MARVELL_ORION_1_88F5182,      4, 24, 2, orion_88f5182_xore_irqs },
        { MARVELL_ORION_2_88F5281,      4, 24, 0, NULL },
        { MARVELL_ORION_1_88W8660,      4, 24, 0, NULL },
        { MARVELL_KIRKWOOD_88F6180,     0, -1, 4, kirkwood_xore_irqs },
        { MARVELL_KIRKWOOD_88F6192,     0, -1, 4, kirkwood_xore_irqs },
        { MARVELL_KIRKWOOD_88F6281,     0, -1, 4, kirkwood_xore_irqs },
        { MARVELL_KIRKWOOD_88F6282,     0, -1, 4, kirkwood_xore_irqs },
        { MARVELL_DOVE_88AP510,         0, -1, 4, dove_xore_irqs },
        { MARVELL_ARMADAXP_MV78130,     4, 33, 2, armadaxp_xore_irqs0 },
        { MARVELL_ARMADAXP_MV78130,     0, -1, 2, armadaxp_xore_irqs1 },
        { MARVELL_ARMADAXP_MV78160,     4, 33, 2, armadaxp_xore_irqs0 },
        { MARVELL_ARMADAXP_MV78160,     0, -1, 2, armadaxp_xore_irqs1 },
        { MARVELL_ARMADAXP_MV78230,     4, 33, 2, armadaxp_xore_irqs0 },
        { MARVELL_ARMADAXP_MV78230,     0, -1, 2, armadaxp_xore_irqs1 },
        { MARVELL_ARMADAXP_MV78260,     4, 33, 2, armadaxp_xore_irqs0 },
        { MARVELL_ARMADAXP_MV78260,     0, -1, 2, armadaxp_xore_irqs1 },
        { MARVELL_ARMADAXP_MV78460,     4, 33, 2, armadaxp_xore_irqs0 },
        { MARVELL_ARMADAXP_MV78460,     0, -1, 2, armadaxp_xore_irqs1 },
};

struct gtidmac_winacctbl *gtidmac_winacctbl;
struct gtidmac_winacctbl *mvxore_winacctbl;

CFATTACH_DECL_NEW(gtidmac_gt, sizeof(struct gtidmac_softc),
    gtidmac_match, gtidmac_attach, NULL, NULL);
CFATTACH_DECL_NEW(gtidmac_mbus, sizeof(struct gtidmac_softc),
    gtidmac_match, gtidmac_attach, NULL, NULL);


/* ARGSUSED */
static int
gtidmac_match(device_t parent, struct cfdata *match, void *aux)
{
        struct marvell_attach_args *mva = aux;
        int unit, i;

        if (strcmp(mva->mva_name, match->cf_name) != 0)
                return 0;
        if (mva->mva_offset == MVA_OFFSET_DEFAULT)
                return 0;
        unit = 0;
        for (i = 0; i < __arraycount(channels); i++)
                if (mva->mva_model == channels[i].model) {
                        if (mva->mva_unit == unit) {
                                mva->mva_size = GTIDMAC_SIZE;
                                return 1;
                        }
                        unit++;
                }
        return 0;
}

/* ARGSUSED */
static void
gtidmac_attach(device_t parent, device_t self, void *aux)
{
        struct gtidmac_softc *sc = device_private(self);
        struct marvell_attach_args *mva = aux;
        prop_dictionary_t dict = device_properties(self);
        uint32_t idmac_irq, xore_irq, *xore_irqs, dmb_speed;
        int unit, idmac_nchan, xore_nchan, nsegs, i, j, n;

        unit = 0;
        for (i = 0; i < __arraycount(channels); i++)
                if (mva->mva_model == channels[i].model) {
                        if (mva->mva_unit == unit)
                                break;
                        unit++;
                }
        idmac_nchan = channels[i].idmac_nchan;
        idmac_irq = channels[i].idmac_irq;
        if (idmac_nchan != 0) {
                if (idmac_irq == -1)
                        idmac_irq = mva->mva_irq;
                if (idmac_irq == -1)
                        /* Discovery */
                        if (!prop_dictionary_get_uint32(dict,
                            "idmac-irq", &idmac_irq)) {
                                aprint_error(": no idmac-irq property\n");
                                return;
                        }
        }
        xore_nchan = channels[i].xore_nchan;
        xore_irqs = channels[i].xore_irqs;
        xore_irq = MVA_IRQ_DEFAULT;
        if (xore_nchan != 0) {
                if (xore_irqs == NULL)
                        xore_irq = mva->mva_irq;
                if (xore_irqs == NULL && xore_irq == MVA_IRQ_DEFAULT)
                        /* Discovery LT/V/VI */
                        if (!prop_dictionary_get_uint32(dict,
                            "xore-irq", &xore_irq)) {
                                aprint_error(": no xore-irq property\n");
                                return;
                        }
        }

        aprint_naive("\n");
        aprint_normal(": Marvell IDMA Controller%s\n",
            xore_nchan ? "/XOR Engine" : "");
        if (idmac_nchan > 0)
                aprint_normal_dev(self,
                    "IDMA Controller %d channels, intr %d...%d\n",
                    idmac_nchan, idmac_irq, idmac_irq + GTIDMAC_NINTRRUPT - 1);
        if (xore_nchan > 0) {
                aprint_normal_dev(self, "XOR Engine %d channels", xore_nchan);
                if (xore_irqs == NULL)
                        aprint_normal(", intr %d...%d\n",
                            xore_irq, xore_irq + xore_nchan - 1);
                else {
                        aprint_normal(", intr %d", xore_irqs[0]);
                        for (i = 1; i < xore_nchan; i++)
                                aprint_normal(", %d", xore_irqs[i]);
                        aprint_normal("\n");
                }
        }

        sc->sc_dev = self;
        sc->sc_iot = mva->mva_iot;

        /* Map I/O registers */
        if (bus_space_subregion(mva->mva_iot, mva->mva_ioh, mva->mva_offset,
            mva->mva_size, &sc->sc_ioh)) {
                aprint_error_dev(self, "can't map registers\n");
                return;
        }

        /*
         * Initialise DMA descriptors and associated metadata
         */
        sc->sc_dmat = mva->mva_dmat;
        n = idmac_nchan * GTIDMAC_NDESC + xore_nchan * MVXORE_NDESC;
        sc->sc_dd_buffer =
            kmem_alloc(sizeof(struct gtidmac_dma_desc) * n, KM_SLEEP);

        /* pattern buffer */
        if (bus_dmamem_alloc(sc->sc_dmat, PAGE_SIZE, PAGE_SIZE, 0,
            &sc->sc_pattern_segment, 1, &nsegs, BUS_DMA_NOWAIT)) {
                aprint_error_dev(self,
                    "bus_dmamem_alloc failed: pattern buffer\n");
                goto fail2;
        }
        if (bus_dmamem_map(sc->sc_dmat, &sc->sc_pattern_segment, 1, PAGE_SIZE,
            (void **)&sc->sc_pbuf, BUS_DMA_NOWAIT)) {
                aprint_error_dev(self,
                    "bus_dmamem_map failed: pattern buffer\n");
                goto fail3;
        }
        for (i = 0; i < 0x100; i++)
                for (j = 0; j < sizeof(sc->sc_pbuf[i].pbuf); j++)
                        sc->sc_pbuf[i].pbuf[j] = i;

        if (!prop_dictionary_get_uint32(dict, "dmb_speed", &dmb_speed)) {
                aprint_error_dev(self, "no dmb_speed property\n");
                dmb_speed = 10; /* More than fast swdmover perhaps. */
        }

        /* IDMAC DMA descriptor buffer */
        sc->sc_gtidmac_nchan = idmac_nchan;
        if (sc->sc_gtidmac_nchan > 0) {
                if (gtidmac_buffer_setup(sc) != 0)
                        goto fail4;

                if (mva->mva_model != MARVELL_DISCOVERY)
                        gtidmac_wininit(sc, mva->mva_tags);

                /* Setup interrupt */
                for (i = 0; i < GTIDMAC_NINTRRUPT; i++) {
                        j = i * idmac_nchan / GTIDMAC_NINTRRUPT;

                        sc->sc_intrarg[i].ia_sc = sc;
                        sc->sc_intrarg[i].ia_cause = GTIDMAC_ICR(j);
                        sc->sc_intrarg[i].ia_eaddr = GTIDMAC_EAR(j);
                        sc->sc_intrarg[i].ia_eselect = GTIDMAC_ESR(j);
                        marvell_intr_establish(idmac_irq + i, IPL_BIO,
                            gtidmac_intr, &sc->sc_intrarg[i]);
                }

                /* Register us with dmover. */
                sc->sc_dmb.dmb_name = device_xname(self);
                sc->sc_dmb.dmb_speed = dmb_speed;
                sc->sc_dmb.dmb_cookie = sc;
                sc->sc_dmb.dmb_algdescs = gtidmac_algdescs;
                sc->sc_dmb.dmb_nalgdescs = __arraycount(gtidmac_algdescs);
                sc->sc_dmb.dmb_process = gtidmac_process;
                dmover_backend_register(&sc->sc_dmb);
                sc->sc_dmb_busy = 0;
        }

        /* XORE DMA descriptor buffer */
        sc->sc_mvxore_nchan = xore_nchan;
        if (sc->sc_mvxore_nchan > 0) {
                if (mvxore_buffer_setup(sc) != 0)
                        goto fail5;

                /* Setup interrupt */
                for (i = 0; i < sc->sc_mvxore_nchan; i++)
                        marvell_intr_establish(
                            xore_irqs != NULL ? xore_irqs[i] : xore_irq + i,
                            IPL_BIO,
                            (i & 0x2) ? mvxore_port1_intr : mvxore_port0_intr,
                            sc);

                mvxore_wininit(sc, mva->mva_tags);

                /* Register us with dmover. */
                sc->sc_dmb_xore.dmb_name = device_xname(sc->sc_dev);
                sc->sc_dmb_xore.dmb_speed = dmb_speed;
                sc->sc_dmb_xore.dmb_cookie = sc;
                sc->sc_dmb_xore.dmb_algdescs = mvxore_algdescs;
                sc->sc_dmb_xore.dmb_nalgdescs = __arraycount(mvxore_algdescs);
                sc->sc_dmb_xore.dmb_process = gtidmac_process;
                dmover_backend_register(&sc->sc_dmb_xore);
        }

        gtidmac_softc = sc;

        return;

fail5:
        for (i = sc->sc_gtidmac_nchan - 1; i >= 0; i--) {
                bus_dmamap_destroy(sc->sc_dmat, sc->sc_cdesc[i].chan_in);
                bus_dmamap_destroy(sc->sc_dmat, sc->sc_cdesc[i].chan_out);
        }
        bus_dmamap_unload(sc->sc_dmat, sc->sc_dmap);
        bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmap);
        bus_dmamem_unmap(sc->sc_dmat, sc->sc_dbuf,
            sizeof(struct gtidmac_desc) * GTIDMAC_NDESC);
        bus_dmamem_free(sc->sc_dmat,
            sc->sc_dmap->dm_segs, sc->sc_dmap->dm_nsegs);
fail4:
        bus_dmamem_unmap(sc->sc_dmat, sc->sc_pbuf, PAGE_SIZE);
fail3:
        bus_dmamem_free(sc->sc_dmat, &sc->sc_pattern_segment, 1);
fail2:
        kmem_free(sc->sc_dd_buffer, sizeof(struct gtidmac_dma_desc) * n);
        bus_space_unmap(sc->sc_iot, sc->sc_ioh, mva->mva_size);
        return;
}


static int
gtidmac_intr(void *arg)
{
        struct gtidmac_intr_arg *ia = arg;
        struct gtidmac_softc *sc = ia->ia_sc;
        uint32_t cause;
        int handled = 0, chan, error;

        cause = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ia->ia_cause);
        DPRINTF(("IDMAC intr: cause=0x%x\n", cause));
        bus_space_write_4(sc->sc_iot, sc->sc_ioh, ia->ia_cause, ~cause);

        chan = 0;
        while (cause) {
                error = 0;
                if (cause & GTIDMAC_I_ADDRMISS) {
                        aprint_error_dev(sc->sc_dev, "Address Miss");
                        error = EINVAL;
                }
                if (cause & GTIDMAC_I_ACCPROT) {
                        aprint_error_dev(sc->sc_dev,
                            "Access Protect Violation");
                        error = EACCES;
                }
                if (cause & GTIDMAC_I_WRPROT) {
                        aprint_error_dev(sc->sc_dev, "Write Protect");
                        error = EACCES;
                }
                if (cause & GTIDMAC_I_OWN) {
                        aprint_error_dev(sc->sc_dev, "Ownership Violation");
                        error = EINVAL;
                }

#define GTIDMAC_I_ERROR           \
           (GTIDMAC_I_ADDRMISS  | \
            GTIDMAC_I_ACCPROT   | \
            GTIDMAC_I_WRPROT    | \
            GTIDMAC_I_OWN)
                if (cause & GTIDMAC_I_ERROR) {
                        uint32_t sel;
                        int select;

                        sel = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                            ia->ia_eselect) & GTIDMAC_ESR_SEL;
                        select = sel - chan * GTIDMAC_I_BITS;
                        if (select >= 0 && select < GTIDMAC_I_BITS) {
                                uint32_t ear;

                                ear = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                                    ia->ia_eaddr);
                                aprint_error(": Error Address 0x%x\n", ear);
                        } else
                                aprint_error(": lost Error Address\n");
                }

                if (cause & (GTIDMAC_I_COMP | GTIDMAC_I_ERROR)) {
                        sc->sc_cdesc[chan].chan_dma_done(
                            sc->sc_cdesc[chan].chan_running, chan,
                            &sc->sc_cdesc[chan].chan_in,
                            &sc->sc_cdesc[chan].chan_out, error);
                        handled++;
                }

                cause >>= GTIDMAC_I_BITS;
        }
        DPRINTF(("IDMAC intr: %shandled\n", handled ? "" : "not "));

        return handled;
}

static int
mvxore_port0_intr(void *arg)
{
        struct gtidmac_softc *sc = arg;

        return mvxore_intr(sc, 0);
}

static int
mvxore_port1_intr(void *arg)
{
        struct gtidmac_softc *sc = arg;

        return mvxore_intr(sc, 1);
}

static int
mvxore_intr(struct gtidmac_softc *sc, int port)
{
        uint32_t cause;
        int handled = 0, chan, error;

        cause =
            bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEICR(sc, port));
        DPRINTF(("XORE port %d intr: cause=0x%x\n", port, cause));
        bus_space_write_4(sc->sc_iot, sc->sc_ioh,
            MVXORE_XEICR(sc, port), ~cause);

        chan = 0;
        while (cause) {
                error = 0;
                if (cause & MVXORE_I_ADDRDECODE) {
                        aprint_error_dev(sc->sc_dev, "Failed address decoding");
                        error = EINVAL;
                }
                if (cause & MVXORE_I_ACCPROT) {
                        aprint_error_dev(sc->sc_dev,
                            "Access Protect Violation");
                        error = EACCES;
                }
                if (cause & MVXORE_I_WRPROT) {
                        aprint_error_dev(sc->sc_dev, "Write Protect");
                        error = EACCES;
                }
                if (cause & MVXORE_I_OWN) {
                        aprint_error_dev(sc->sc_dev, "Ownership Violation");
                        error = EINVAL;
                }
                if (cause & MVXORE_I_INTPARITY) {
                        aprint_error_dev(sc->sc_dev, "Parity Error");
                        error = EIO;
                }
                if (cause & MVXORE_I_XBAR) {
                        aprint_error_dev(sc->sc_dev, "Crossbar Parity Error");
                        error = EINVAL;
                }

#define MVXORE_I_ERROR            \
           (MVXORE_I_ADDRDECODE | \
            MVXORE_I_ACCPROT    | \
            MVXORE_I_WRPROT     | \
            MVXORE_I_OWN        | \
            MVXORE_I_INTPARITY  | \
            MVXORE_I_XBAR)
                if (cause & MVXORE_I_ERROR) {
                        uint32_t type;
                        int event;

                        type = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                            MVXORE_XEECR(sc, port));
                        type &= MVXORE_XEECR_ERRORTYPE_MASK;
                        event = type - chan * MVXORE_I_BITS;
                        if (event >= 0 && event < MVXORE_I_BITS) {
                                uint32_t xeear;

                                xeear = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                                    MVXORE_XEEAR(sc, port));
                                aprint_error(": Error Address 0x%x\n", xeear);
                        } else
                                aprint_error(": lost Error Address\n");
                }

                if (cause & (MVXORE_I_EOC | MVXORE_I_ERROR)) {
                        sc->sc_cdesc_xore[chan].chan_dma_done(
                            sc->sc_cdesc_xore[chan].chan_running, chan,
                            sc->sc_cdesc_xore[chan].chan_in,
                            &sc->sc_cdesc_xore[chan].chan_out, error);
                        handled++;
                }

                cause >>= MVXORE_I_BITS;
        }
        DPRINTF(("XORE port %d intr: %shandled\n",
            port, handled ? "" : "not "));

        return handled;
}


/*
 * dmover(9) backend function.
 */
static void
gtidmac_process(struct dmover_backend *dmb)
{
        struct gtidmac_softc *sc = dmb->dmb_cookie;
        int s;

        /* If the backend is currently idle, go process the queue. */
        s = splbio();
        if (!sc->sc_dmb_busy)
                gtidmac_dmover_run(dmb);
        splx(s);
}

static void
gtidmac_dmover_run(struct dmover_backend *dmb)
{
        struct gtidmac_softc *sc = dmb->dmb_cookie;
        struct dmover_request *dreq;
        const struct dmover_algdesc *algdesc;
        struct gtidmac_function *df;
        bus_dmamap_t *dmamap_in, *dmamap_out;
        int chan, ninputs, error, i;

        sc->sc_dmb_busy = 1;

        for (;;) {
                dreq = TAILQ_FIRST(&dmb->dmb_pendreqs);
                if (dreq == NULL)
                        break;
                algdesc = dreq->dreq_assignment->das_algdesc;
                df = algdesc->dad_data;
                chan = (*df->chan_alloc)(sc, &dmamap_in, &dmamap_out, dreq);
                if (chan == -1)
                        return;

                dmover_backend_remque(dmb, dreq);
                dreq->dreq_flags |= DMOVER_REQ_RUNNING;

                /* XXXUNLOCK */

                error = 0;

                /* Load in/out buffers of dmover to bus_dmamap. */
                ninputs = dreq->dreq_assignment->das_algdesc->dad_ninputs;
                if (ninputs == 0) {
                        int pno = 0;

                        if (algdesc->dad_name == DMOVER_FUNC_FILL8)
                                pno = dreq->dreq_immediate[0];

                        i = 0;
                        error = bus_dmamap_load(sc->sc_dmat, *dmamap_in,
                            &sc->sc_pbuf[pno], sizeof(sc->sc_pbuf[pno]), NULL,
                            BUS_DMA_NOWAIT | BUS_DMA_STREAMING | BUS_DMA_WRITE);
                        if (error == 0) {
                                bus_dmamap_sync(sc->sc_dmat, *dmamap_in, 0,
                                    sizeof(sc->sc_pbuf[pno]),
                                    BUS_DMASYNC_PREWRITE);

                                /*
                                 * We will call gtidmac_dmmap_unload() when
                                 * becoming an error.
                                 */
                                i = 1;
                        }
                } else
                        for (i = 0; i < ninputs; i++) {
                                error = gtidmac_dmmap_load(sc,
                                    *(dmamap_in + i), dreq->dreq_inbuf_type,
                                    &dreq->dreq_inbuf[i], 0/*write*/);
                                if (error != 0)
                                        break;
                        }
                if (algdesc->dad_name != DMOVER_FUNC_ISCSI_CRC32C) {
                        if (error == 0)
                                error = gtidmac_dmmap_load(sc, *dmamap_out,
                                    dreq->dreq_outbuf_type, &dreq->dreq_outbuf,
                                    1/*read*/);

                        if (error == 0) {
                                /*
                                 * The size of outbuf is always believed to be
                                 * DMA transfer size in dmover request.
                                 */
                                error = (*df->dma_setup)(sc, chan, ninputs,
                                    dmamap_in, dmamap_out,
                                    (*dmamap_out)->dm_mapsize);
                                if (error != 0)
                                        gtidmac_dmmap_unload(sc, *dmamap_out,
                                            1);
                        }
                } else
                        if (error == 0)
                                error = (*df->dma_setup)(sc, chan, ninputs,
                                    dmamap_in, dmamap_out,
                                    (*dmamap_in)->dm_mapsize);

                /* XXXLOCK */

                if (error != 0) {
                        for (; i-- > 0;)
                                gtidmac_dmmap_unload(sc, *(dmamap_in + i), 0);
                        (*df->chan_free)(sc, chan);

                        dreq->dreq_flags |= DMOVER_REQ_ERROR;
                        dreq->dreq_error = error;
                        /* XXXUNLOCK */
                        dmover_done(dreq);
                        /* XXXLOCK */
                        continue;
                }

                (*df->dma_start)(sc, chan, gtidmac_dmover_done);
                break;
        }

        /* All done */
        sc->sc_dmb_busy = 0;
}

static void
gtidmac_dmover_done(void *object, int chan, bus_dmamap_t *dmamap_in,
                    bus_dmamap_t *dmamap_out, int error)
{
        struct gtidmac_softc *sc;
        struct dmover_request *dreq = object;
        struct dmover_backend *dmb;
        struct gtidmac_function *df;
        uint32_t result;
        int ninputs, i;

        KASSERT(dreq != NULL);

        dmb = dreq->dreq_assignment->das_backend;
        df = dreq->dreq_assignment->das_algdesc->dad_data;
        ninputs = dreq->dreq_assignment->das_algdesc->dad_ninputs;
        sc = dmb->dmb_cookie;

        result = (*df->dma_finish)(sc, chan, error);
        for (i = 0; i < ninputs; i++)
                gtidmac_dmmap_unload(sc, *(dmamap_in + i), 0);
        if (dreq->dreq_assignment->das_algdesc->dad_name ==
            DMOVER_FUNC_ISCSI_CRC32C)
                memcpy(dreq->dreq_immediate, &result, sizeof(result));
        else
                gtidmac_dmmap_unload(sc, *dmamap_out, 1);

        (*df->chan_free)(sc, chan);

        if (error) {
                dreq->dreq_error = error;
                dreq->dreq_flags |= DMOVER_REQ_ERROR;
        }

        dmover_done(dreq);

        /*
         * See if we can start some more dmover(9) requests.
         *
         * Note: We're already at splbio() here.
         */
        if (!sc->sc_dmb_busy)
                gtidmac_dmover_run(dmb);
}

static __inline int
gtidmac_dmmap_load(struct gtidmac_softc *sc, bus_dmamap_t dmamap,
                   dmover_buffer_type dmbuf_type, dmover_buffer *dmbuf,
                   int read)
{
        int error, flags;

        flags = BUS_DMA_NOWAIT | BUS_DMA_STREAMING |
            read ? BUS_DMA_READ : BUS_DMA_WRITE;

        switch (dmbuf_type) {
        case DMOVER_BUF_LINEAR:
                error = bus_dmamap_load(sc->sc_dmat, dmamap,
                    dmbuf->dmbuf_linear.l_addr, dmbuf->dmbuf_linear.l_len,
                    NULL, flags);
                break;

        case DMOVER_BUF_UIO:
                if ((read && dmbuf->dmbuf_uio->uio_rw != UIO_READ) ||
                    (!read && dmbuf->dmbuf_uio->uio_rw == UIO_READ))
                        return (EINVAL);

                error = bus_dmamap_load_uio(sc->sc_dmat, dmamap,
                    dmbuf->dmbuf_uio, flags);
                break;

        default:
                error = EINVAL;
        }

        if (error == 0)
                bus_dmamap_sync(sc->sc_dmat, dmamap, 0, dmamap->dm_mapsize,
                    read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);

        return error;
}

static __inline void
gtidmac_dmmap_unload(struct gtidmac_softc *sc, bus_dmamap_t dmamap, int read)
{

        bus_dmamap_sync(sc->sc_dmat, dmamap, 0, dmamap->dm_mapsize,
            read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);

        bus_dmamap_unload(sc->sc_dmat, dmamap);
}


/*
 * IDMAC functions
 */
int
gtidmac_chan_alloc(void *tag, bus_dmamap_t **dmamap_in,
                   bus_dmamap_t **dmamap_out, void *object)
{
        struct gtidmac_softc *sc = tag;
        int chan;

/* maybe need lock */

        for (chan = 0; chan < sc->sc_gtidmac_nchan; chan++)
                if (sc->sc_cdesc[chan].chan_running == NULL)
                        break;
        if (chan >= sc->sc_gtidmac_nchan)
                return -1;


        sc->sc_cdesc[chan].chan_running = object;

/* unlock */

        *dmamap_in = &sc->sc_cdesc[chan].chan_in;
        *dmamap_out = &sc->sc_cdesc[chan].chan_out;

        return chan;
}

void
gtidmac_chan_free(void *tag, int chan)
{
        struct gtidmac_softc *sc = tag;

/* maybe need lock */

        sc->sc_cdesc[chan].chan_running = NULL;

/* unlock */
}

/* ARGSUSED */
int
gtidmac_setup(void *tag, int chan, int ninputs, bus_dmamap_t *dmamap_in,
              bus_dmamap_t *dmamap_out, bus_size_t size)
{
        struct gtidmac_softc *sc = tag;
        struct gtidmac_dma_desc *dd, *fstdd, *nxtdd;
        struct gtidmac_desc *desc;
        uint32_t ccl, bcnt, ires, ores;
        int n = 0, iidx, oidx;

        KASSERT(ninputs == 0 || ninputs == 1);

        ccl = bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCLR(chan));
#ifdef DIAGNOSTIC
        if (ccl & GTIDMAC_CCLR_CHANACT)
                panic("gtidmac_setup: chan%d already active", chan);
#endif

        /* We always Chain-mode and max (16M - 1)byte/desc */
        ccl = (GTIDMAC_CCLR_DESCMODE_16M                                |
#ifdef GTIDMAC_DEBUG
            GTIDMAC_CCLR_CDEN                                           |
#endif
            GTIDMAC_CCLR_TRANSFERMODE_B /* Transfer Mode: Block */      |
            GTIDMAC_CCLR_INTMODE_NULL   /* Intr Mode: Next Desc NULL */ |
            GTIDMAC_CCLR_CHAINMODE_C    /* Chain Mode: Chaind */);
        if (size != (*dmamap_in)->dm_mapsize) {
                ccl |= GTIDMAC_CCLR_SRCHOLD;
                if ((*dmamap_in)->dm_mapsize == 8)
                        ccl |= GTIDMAC_CCLR_SBL_8B;
                else if ((*dmamap_in)->dm_mapsize == 16)
                        ccl |= GTIDMAC_CCLR_SBL_16B;
                else if ((*dmamap_in)->dm_mapsize == 32)
                        ccl |= GTIDMAC_CCLR_SBL_32B;
                else if ((*dmamap_in)->dm_mapsize == 64)
                        ccl |= GTIDMAC_CCLR_SBL_64B;
                else if ((*dmamap_in)->dm_mapsize == 128)
                        ccl |= GTIDMAC_CCLR_SBL_128B;
                else
                        panic("gtidmac_setup: chan%d source:"
                            " unsupported hold size", chan);
        } else
                ccl |= GTIDMAC_CCLR_SBL_128B;
        if (size != (*dmamap_out)->dm_mapsize) {
                ccl |= GTIDMAC_CCLR_DESTHOLD;
                if ((*dmamap_out)->dm_mapsize == 8)
                        ccl |= GTIDMAC_CCLR_DBL_8B;
                else if ((*dmamap_out)->dm_mapsize == 16)
                        ccl |= GTIDMAC_CCLR_DBL_16B;
                else if ((*dmamap_out)->dm_mapsize == 32)
                        ccl |= GTIDMAC_CCLR_DBL_32B;
                else if ((*dmamap_out)->dm_mapsize == 64)
                        ccl |= GTIDMAC_CCLR_DBL_64B;
                else if ((*dmamap_out)->dm_mapsize == 128)
                        ccl |= GTIDMAC_CCLR_DBL_128B;
                else
                        panic("gtidmac_setup: chan%d destination:"
                            " unsupport hold size", chan);
        } else
                ccl |= GTIDMAC_CCLR_DBL_128B;

        fstdd = SLIST_FIRST(&sc->sc_dlist);
        if (fstdd == NULL) {
                aprint_error_dev(sc->sc_dev, "no descriptor\n");
                return ENOMEM;
        }
        SLIST_REMOVE_HEAD(&sc->sc_dlist, dd_next);
        sc->sc_cdesc[chan].chan_ddidx = fstdd->dd_index;

        dd = fstdd;
        ires = ores = 0;
        iidx = oidx = 0;
        while (1 /*CONSTCOND*/) {
                if (ccl & GTIDMAC_CCLR_SRCHOLD) {
                        if (ccl & GTIDMAC_CCLR_DESTHOLD)
                                bcnt = size;    /* src/dst hold */
                        else
                                bcnt = (*dmamap_out)->dm_segs[oidx].ds_len;
                } else if (ccl & GTIDMAC_CCLR_DESTHOLD)
                        bcnt = (*dmamap_in)->dm_segs[iidx].ds_len;
                else
                        bcnt = uimin((*dmamap_in)->dm_segs[iidx].ds_len - ires,
                            (*dmamap_out)->dm_segs[oidx].ds_len - ores);

                desc = dd->dd_idmac_vaddr;
                desc->bc.mode16m.bcnt =
                    bcnt | GTIDMAC_CIDMABCR_BCLEFT | GTIDMAC_CIDMABCR_OWN;
                desc->srcaddr = (*dmamap_in)->dm_segs[iidx].ds_addr + ires;
                desc->dstaddr = (*dmamap_out)->dm_segs[oidx].ds_addr + ores;

                n += bcnt;
                if (n >= size)
                        break;
                if (!(ccl & GTIDMAC_CCLR_SRCHOLD)) {
                        ires += bcnt;
                        if (ires >= (*dmamap_in)->dm_segs[iidx].ds_len) {
                                ires = 0;
                                iidx++;
                                KASSERT(iidx < (*dmamap_in)->dm_nsegs);
                        }
                }
                if (!(ccl & GTIDMAC_CCLR_DESTHOLD)) {
                        ores += bcnt;
                        if (ores >= (*dmamap_out)->dm_segs[oidx].ds_len) {
                                ores = 0;
                                oidx++;
                                KASSERT(oidx < (*dmamap_out)->dm_nsegs);
                        }
                }

                nxtdd = SLIST_FIRST(&sc->sc_dlist);
                if (nxtdd == NULL) {
                        aprint_error_dev(sc->sc_dev, "no descriptor\n");
                        return ENOMEM;
                }
                SLIST_REMOVE_HEAD(&sc->sc_dlist, dd_next);

                desc->nextdp = (uint32_t)nxtdd->dd_paddr;
                bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap,
                    dd->dd_index * sizeof(*desc), sizeof(*desc),
#ifdef GTIDMAC_DEBUG
                    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
#else
                    BUS_DMASYNC_PREWRITE);
#endif

                SLIST_INSERT_AFTER(dd, nxtdd, dd_next);
                dd = nxtdd;
        }
        desc->nextdp = (uint32_t)NULL;
        bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap, dd->dd_index * sizeof(*desc),
#ifdef GTIDMAC_DEBUG
            sizeof(*desc), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
#else
            sizeof(*desc), BUS_DMASYNC_PREWRITE);
#endif

        /* Set paddr of descriptor to Channel Next Descriptor Pointer */
        bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CNDPR(chan),
            fstdd->dd_paddr);

#if BYTE_ORDER == LITTLE_ENDIAN
        bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCHR(chan),
            GTIDMAC_CCHR_DESCBYTESWAP | GTIDMAC_CCHR_ENDIAN_LE);
#else
        bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCHR(chan),
            GTIDMAC_CCHR_DESCBYTESWAP | GTIDMAC_CCHR_ENDIAN_BE);
#endif
        bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCLR(chan), ccl);

#ifdef GTIDMAC_DEBUG
        gtidmac_dump_idmacdesc(sc, fstdd, ccl, 0/*pre*/);
#endif

        sc->sc_cdesc[chan].chan_totalcnt += size;

        return 0;
}

void
gtidmac_start(void *tag, int chan,
              void (*dma_done_cb)(void *, int, bus_dmamap_t *, bus_dmamap_t *,
                                  int))
{
        struct gtidmac_softc *sc = tag;
        uint32_t ccl;

        DPRINTF(("%s:%d: starting\n", device_xname(sc->sc_dev), chan));

#ifdef GTIDMAC_DEBUG
        gtidmac_dump_idmacreg(sc, chan);
#endif

        sc->sc_cdesc[chan].chan_dma_done = dma_done_cb;

        ccl = bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCLR(chan));
        /* Start and 'Fetch Next Descriptor' */
        bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCLR(chan),
            ccl | GTIDMAC_CCLR_CHANEN | GTIDMAC_CCLR_FETCHND);
}

static uint32_t
gtidmac_finish(void *tag, int chan, int error)
{
        struct gtidmac_softc *sc = tag;
        struct gtidmac_dma_desc *dd, *fstdd, *nxtdd;
        struct gtidmac_desc *desc;

        fstdd = &sc->sc_dd_buffer[sc->sc_cdesc[chan].chan_ddidx];

#ifdef GTIDMAC_DEBUG
        if (error || gtidmac_debug > 1) {
                uint32_t ccl;

                gtidmac_dump_idmacreg(sc, chan);
                ccl = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                    GTIDMAC_CCLR(chan));
                gtidmac_dump_idmacdesc(sc, fstdd, ccl, 1/*post*/);
        }
#endif

        dd = fstdd;
        do {
                desc = dd->dd_idmac_vaddr;

                bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap,
                    dd->dd_index * sizeof(*desc), sizeof(*desc),
#ifdef GTIDMAC_DEBUG
                    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
#else
                    BUS_DMASYNC_POSTWRITE);
#endif

                nxtdd = SLIST_NEXT(dd, dd_next);
                SLIST_INSERT_HEAD(&sc->sc_dlist, dd, dd_next);
                dd = nxtdd;
        } while (desc->nextdp);

        return 0;
}

/*
 * XORE functions
 */
int
mvxore_chan_alloc(void *tag, bus_dmamap_t **dmamap_in,
                  bus_dmamap_t **dmamap_out, void *object)
{
        struct gtidmac_softc *sc = tag;
        int chan;

/* maybe need lock */

        for (chan = 0; chan < sc->sc_mvxore_nchan; chan++)
                if (sc->sc_cdesc_xore[chan].chan_running == NULL)
                        break;
        if (chan >= sc->sc_mvxore_nchan)
                return -1;


        sc->sc_cdesc_xore[chan].chan_running = object;

/* unlock */

        *dmamap_in = sc->sc_cdesc_xore[chan].chan_in;
        *dmamap_out = &sc->sc_cdesc_xore[chan].chan_out;

        return chan;
}

void
mvxore_chan_free(void *tag, int chan)
{
        struct gtidmac_softc *sc = tag;

/* maybe need lock */

        sc->sc_cdesc_xore[chan].chan_running = NULL;

/* unlock */
}

/* ARGSUSED */
int
mvxore_setup(void *tag, int chan, int ninputs, bus_dmamap_t *dmamap_in,
             bus_dmamap_t *dmamap_out, bus_size_t size)
{
        struct gtidmac_softc *sc = tag;
        struct gtidmac_dma_desc *dd, *fstdd, *nxtdd;
        struct mvxore_desc *desc;
        uint32_t xexc, bcnt, cmd, lastcmd;
        int n = 0, i;
        uint32_t ires[MVXORE_NSRC] = { 0, 0, 0, 0, 0, 0, 0, 0 }, ores = 0;
        int iidx[MVXORE_NSRC] = { 0, 0, 0, 0, 0, 0, 0, 0 }, oidx = 0;

#ifdef DIAGNOSTIC
        uint32_t xexact =
            bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXACTR(sc, chan));

        if ((xexact & MVXORE_XEXACTR_XESTATUS_MASK) ==
            MVXORE_XEXACTR_XESTATUS_ACT)
                panic("mvxore_setup: chan%d already active."
                    " mvxore not support hot insertion", chan);
#endif

        xexc =
            (MVXORE_XEXCR_REGACCPROTECT |
             MVXORE_XEXCR_DBL_128B      |
             MVXORE_XEXCR_SBL_128B);
        cmd = lastcmd = 0;
        if (ninputs > 1) {
                xexc |= MVXORE_XEXCR_OM_XOR;
                lastcmd = cmd = (1 << ninputs) - 1;
        } else if (ninputs == 1) {
                if ((*dmamap_out)->dm_nsegs == 0) {
                        xexc |= MVXORE_XEXCR_OM_CRC32;
                        lastcmd = MVXORE_DESC_CMD_CRCLAST;
                } else
                        xexc |= MVXORE_XEXCR_OM_DMA;
        } else if (ninputs == 0) {
                if ((*dmamap_out)->dm_nsegs != 1) {
                        aprint_error_dev(sc->sc_dev,
                            "XORE not supports %d DMA segments\n",
                            (*dmamap_out)->dm_nsegs);
                        return EINVAL;
                }

                if ((*dmamap_in)->dm_mapsize == 0) {
                        xexc |= MVXORE_XEXCR_OM_ECC;

                        /* XXXXX: Maybe need to set Timer Mode registers? */

#if 0
                } else if ((*dmamap_in)->dm_mapsize == 8 ||
                    (*dmamap_in)->dm_mapsize == 16) { /* in case dmover */
                        uint64_t pattern;

                        /* XXXX: Get pattern data */

                        KASSERT((*dmamap_in)->dm_mapsize == 8 ||
                            (void *)((uint32_t)(*dmamap_in)->_dm_origbuf &
                                                ~PAGE_MASK) == sc->sc_pbuf);
                        pattern = *(uint64_t *)(*dmamap_in)->_dm_origbuf;

                        /* XXXXX: XORE has a IVR.  We should get this first. */
                        bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEIVRL,
                            pattern);
                        bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEIVRH,
                            pattern >> 32);

                        xexc |= MVXORE_XEXCR_OM_MEMINIT;
#endif
                } else {
                        aprint_error_dev(sc->sc_dev,
                            "XORE not supports DMA mapsize %zd\n",
                            (*dmamap_in)->dm_mapsize);
                        return EINVAL;
                }
                bus_space_write_4(sc->sc_iot, sc->sc_ioh,
                    MVXORE_XEXDPR(sc, chan), (*dmamap_out)->dm_segs[0].ds_addr);
                bus_space_write_4(sc->sc_iot, sc->sc_ioh,
                    MVXORE_XEXBSR(sc, chan), (*dmamap_out)->dm_mapsize);

                bus_space_write_4(sc->sc_iot, sc->sc_ioh,
                    MVXORE_XEXCR(sc, chan), xexc);
                sc->sc_cdesc_xore[chan].chan_totalcnt += size;

                return 0;
        }

        /* Make descriptor for DMA/CRC32/XOR */

        fstdd = SLIST_FIRST(&sc->sc_dlist_xore);
        if (fstdd == NULL) {
                aprint_error_dev(sc->sc_dev, "no xore descriptor\n");
                return ENOMEM;
        }
        SLIST_REMOVE_HEAD(&sc->sc_dlist_xore, dd_next);
        sc->sc_cdesc_xore[chan].chan_ddidx =
            fstdd->dd_index + GTIDMAC_NDESC * sc->sc_gtidmac_nchan;

        dd = fstdd;
        while (1 /*CONSTCOND*/) {
                desc = dd->dd_xore_vaddr;
                desc->stat = MVXORE_DESC_STAT_OWN;
                desc->cmd = cmd;
                if ((*dmamap_out)->dm_nsegs != 0) {
                        desc->dstaddr =
                            (*dmamap_out)->dm_segs[oidx].ds_addr + ores;
                        bcnt = (*dmamap_out)->dm_segs[oidx].ds_len - ores;
                } else {
                        desc->dstaddr = 0;
                        bcnt = MVXORE_MAXXFER;  /* XXXXX */
                }
                for (i = 0; i < ninputs; i++) {
                        desc->srcaddr[i] =
                            (*dmamap_in[i]).dm_segs[iidx[i]].ds_addr + ires[i];
                        bcnt = uimin(bcnt,
                            (*dmamap_in[i]).dm_segs[iidx[i]].ds_len - ires[i]);
                }
                desc->bcnt = bcnt;

                n += bcnt;
                if (n >= size)
                        break;
                ores += bcnt;
                if ((*dmamap_out)->dm_nsegs != 0 &&
                    ores >= (*dmamap_out)->dm_segs[oidx].ds_len) {
                        ores = 0;
                        oidx++;
                        KASSERT(oidx < (*dmamap_out)->dm_nsegs);
                }
                for (i = 0; i < ninputs; i++) {
                        ires[i] += bcnt;
                        if (ires[i] >=
                            (*dmamap_in[i]).dm_segs[iidx[i]].ds_len) {
                                ires[i] = 0;
                                iidx[i]++;
                                KASSERT(iidx[i] < (*dmamap_in[i]).dm_nsegs);
                        }
                }

                nxtdd = SLIST_FIRST(&sc->sc_dlist_xore);
                if (nxtdd == NULL) {
                        aprint_error_dev(sc->sc_dev, "no xore descriptor\n");
                        return ENOMEM;
                }
                SLIST_REMOVE_HEAD(&sc->sc_dlist_xore, dd_next);

                desc->nextda = (uint32_t)nxtdd->dd_paddr;
                bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap_xore,
                    dd->dd_index * sizeof(*desc), sizeof(*desc),
                    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

                SLIST_INSERT_AFTER(dd, nxtdd, dd_next);
                dd = nxtdd;
        }
        desc->cmd = lastcmd;
        desc->nextda = (uint32_t)NULL;
        bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap_xore,
            dd->dd_index * sizeof(*desc), sizeof(*desc),
            BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

        /* Set paddr of descriptor to Channel Next Descriptor Pointer */
        bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXNDPR(sc, chan),
            fstdd->dd_paddr);

        bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXCR(sc, chan), xexc);

#ifdef GTIDMAC_DEBUG
        gtidmac_dump_xoredesc(sc, fstdd, xexc, 0/*pre*/);
#endif

        sc->sc_cdesc_xore[chan].chan_totalcnt += size;

        return 0;
}

void
mvxore_start(void *tag, int chan,
             void (*dma_done_cb)(void *, int, bus_dmamap_t *, bus_dmamap_t *,
                                 int))
{
        struct gtidmac_softc *sc = tag;
        uint32_t xexact;

        DPRINTF(("%s:%d: xore starting\n", device_xname(sc->sc_dev), chan));

#ifdef GTIDMAC_DEBUG
        gtidmac_dump_xorereg(sc, chan);
#endif

        sc->sc_cdesc_xore[chan].chan_dma_done = dma_done_cb;

        xexact =
            bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXACTR(sc, chan));
        bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXACTR(sc, chan),
            xexact | MVXORE_XEXACTR_XESTART);
}

static uint32_t
mvxore_finish(void *tag, int chan, int error)
{
        struct gtidmac_softc *sc = tag;
        struct gtidmac_dma_desc *dd, *fstdd, *nxtdd;
        struct mvxore_desc *desc;
        uint32_t xexc;

#ifdef GTIDMAC_DEBUG
        if (error || gtidmac_debug > 1)
                gtidmac_dump_xorereg(sc, chan);
#endif

        xexc = bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXCR(sc, chan));
        if ((xexc & MVXORE_XEXCR_OM_MASK) == MVXORE_XEXCR_OM_ECC ||
            (xexc & MVXORE_XEXCR_OM_MASK) == MVXORE_XEXCR_OM_MEMINIT)
                return 0;

        fstdd = &sc->sc_dd_buffer[sc->sc_cdesc_xore[chan].chan_ddidx];

#ifdef GTIDMAC_DEBUG
        if (error || gtidmac_debug > 1)
                gtidmac_dump_xoredesc(sc, fstdd, xexc, 1/*post*/);
#endif

        dd = fstdd;
        do {
                desc = dd->dd_xore_vaddr;

                bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap_xore,
                    dd->dd_index * sizeof(*desc), sizeof(*desc),
                    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);

                nxtdd = SLIST_NEXT(dd, dd_next);
                SLIST_INSERT_HEAD(&sc->sc_dlist_xore, dd, dd_next);
                dd = nxtdd;
        } while (desc->nextda);

        if ((xexc & MVXORE_XEXCR_OM_MASK) == MVXORE_XEXCR_OM_CRC32)
                return desc->result;
        return 0;
}

static void
gtidmac_wininit(struct gtidmac_softc *sc, enum marvell_tags *tags)
{
        device_t pdev = device_parent(sc->sc_dev);
        uint64_t base;
        uint32_t size, cxap, en, winacc;
        int window, target, attr, rv, i, j;

        en = 0xff;
        cxap = 0;
        for (window = 0, i = 0;
            tags[i] != MARVELL_TAG_UNDEFINED && window < GTIDMAC_NWINDOW; i++) {
                rv = marvell_winparams_by_tag(pdev, tags[i],
                    &target, &attr, &base, &size);
                if (rv != 0 || size == 0)
                        continue;

                if (base > 0xffffffffULL) {
                        if (window >= GTIDMAC_NREMAP) {
                                aprint_error_dev(sc->sc_dev,
                                    "can't remap window %d\n", window);
                                continue;
                        }
                        bus_space_write_4(sc->sc_iot, sc->sc_ioh,
                            GTIDMAC_HARXR(window), (base >> 32) & 0xffffffff);
                }
                bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_BARX(window),
                    GTIDMAC_BARX_TARGET(target) |
                    GTIDMAC_BARX_ATTR(attr)     |
                    GTIDMAC_BARX_BASE(base));
                bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_SRX(window),
                    GTIDMAC_SRX_SIZE(size));
                en &= ~GTIDMAC_BAER_EN(window);

                winacc = GTIDMAC_CXAPR_WINACC_FA;
                if (gtidmac_winacctbl != NULL)
                        for (j = 0;
                            gtidmac_winacctbl[j].tag != MARVELL_TAG_UNDEFINED;
                            j++) {
                                if (gtidmac_winacctbl[j].tag != tags[i])
                                        continue;

                                switch (gtidmac_winacctbl[j].winacc) {
                                case GTIDMAC_WINACC_NOACCESSALLOWED:
                                        winacc = GTIDMAC_CXAPR_WINACC_NOAA;
                                        break;
                                case GTIDMAC_WINACC_READONLY:
                                        winacc = GTIDMAC_CXAPR_WINACC_RO;
                                        break;
                                case GTIDMAC_WINACC_FULLACCESS:
                                default: /* XXXX: default is full access */
                                        break;
                                }
                                break;
                        }
                cxap |= GTIDMAC_CXAPR_WINACC(window, winacc);

                window++;
        }
        bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_BAER, en);

        for (i = 0; i < GTIDMAC_NACCPROT; i++)
                bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CXAPR(i),
                    cxap);
}

static void
mvxore_wininit(struct gtidmac_softc *sc, enum marvell_tags *tags)
{
        device_t pdev = device_parent(sc->sc_dev);
        uint64_t base;
        uint32_t target, attr, size, xexwc, winacc;
        int window, rv, i, j, p;

        xexwc = 0;
        for (window = 0, i = 0;
            tags[i] != MARVELL_TAG_UNDEFINED && window < MVXORE_NWINDOW; i++) {
                rv = marvell_winparams_by_tag(pdev, tags[i],
                    &target, &attr, &base, &size);
                if (rv != 0 || size == 0)
                        continue;

                if (base > 0xffffffffULL) {
                        if (window >= MVXORE_NREMAP) {
                                aprint_error_dev(sc->sc_dev,
                                    "can't remap window %d\n", window);
                                continue;
                        }
                        for (p = 0; p < sc->sc_mvxore_nchan >> 1; p++)
                                bus_space_write_4(sc->sc_iot, sc->sc_ioh,
                                    MVXORE_XEHARRX(sc, p, window),
                                    (base >> 32) & 0xffffffff);
                }

                for (p = 0; p < sc->sc_mvxore_nchan >> 1; p++) {
                        bus_space_write_4(sc->sc_iot, sc->sc_ioh,
                            MVXORE_XEBARX(sc, p, window),
                            MVXORE_XEBARX_TARGET(target) |
                            MVXORE_XEBARX_ATTR(attr) |
                            MVXORE_XEBARX_BASE(base));
                        bus_space_write_4(sc->sc_iot, sc->sc_ioh,
                            MVXORE_XESMRX(sc, p, window),
                            MVXORE_XESMRX_SIZE(size));
                }

                winacc = MVXORE_XEXWCR_WINACC_FA;
                if (mvxore_winacctbl != NULL)
                        for (j = 0;
                            mvxore_winacctbl[j].tag != MARVELL_TAG_UNDEFINED;
                            j++) {
                                if (gtidmac_winacctbl[j].tag != tags[i])
                                        continue;

                                switch (gtidmac_winacctbl[j].winacc) {
                                case GTIDMAC_WINACC_NOACCESSALLOWED:
                                        winacc = MVXORE_XEXWCR_WINACC_NOAA;
                                        break;
                                case GTIDMAC_WINACC_READONLY:
                                        winacc = MVXORE_XEXWCR_WINACC_RO;
                                        break;
                                case GTIDMAC_WINACC_FULLACCESS:
                                default: /* XXXX: default is full access */
                                        break;
                                }
                                break;
                        }
                xexwc |= (MVXORE_XEXWCR_WINEN(window) |
                    MVXORE_XEXWCR_WINACC(window, winacc));
                window++;
        }

        for (i = 0; i < sc->sc_mvxore_nchan; i++) {
                bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXWCR(sc, i),
                    xexwc);

                /* XXXXX: reset... */
                bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXAOCR(sc, 0),
                    0);
        }
}

static int
gtidmac_buffer_setup(struct gtidmac_softc *sc)
{
        bus_dma_segment_t segs;
        struct gtidmac_dma_desc *dd;
        uint32_t mask;
        int nchan, nsegs, i;

        nchan = sc->sc_gtidmac_nchan;

        if (bus_dmamem_alloc(sc->sc_dmat,
            sizeof(struct gtidmac_desc) * GTIDMAC_NDESC * nchan,
            PAGE_SIZE, 0, &segs, 1, &nsegs, BUS_DMA_NOWAIT)) {
                aprint_error_dev(sc->sc_dev,
                    "bus_dmamem_alloc failed: descriptor buffer\n");
                goto fail0;
        }
        if (bus_dmamem_map(sc->sc_dmat, &segs, 1,
            sizeof(struct gtidmac_desc) * GTIDMAC_NDESC * nchan,
            (void **)&sc->sc_dbuf, BUS_DMA_NOWAIT)) {
                aprint_error_dev(sc->sc_dev,
                    "bus_dmamem_map failed: descriptor buffer\n");
                goto fail1;
        }
        if (bus_dmamap_create(sc->sc_dmat,
            sizeof(struct gtidmac_desc) * GTIDMAC_NDESC * nchan, 1,
            sizeof(struct gtidmac_desc) * GTIDMAC_NDESC * nchan, 0,
            BUS_DMA_NOWAIT, &sc->sc_dmap)) {
                aprint_error_dev(sc->sc_dev,
                    "bus_dmamap_create failed: descriptor buffer\n");
                goto fail2;
        }
        if (bus_dmamap_load(sc->sc_dmat, sc->sc_dmap, sc->sc_dbuf,
            sizeof(struct gtidmac_desc) * GTIDMAC_NDESC * nchan,
            NULL, BUS_DMA_NOWAIT)) {
                aprint_error_dev(sc->sc_dev,
                    "bus_dmamap_load failed: descriptor buffer\n");
                goto fail3;
        }
        SLIST_INIT(&sc->sc_dlist);
        for (i = 0; i < GTIDMAC_NDESC * nchan; i++) {
                dd = &sc->sc_dd_buffer[i];
                dd->dd_index = i;
                dd->dd_idmac_vaddr = &sc->sc_dbuf[i];
                dd->dd_paddr = sc->sc_dmap->dm_segs[0].ds_addr +
                    (sizeof(struct gtidmac_desc) * i);
                        SLIST_INSERT_HEAD(&sc->sc_dlist, dd, dd_next);
        }

        /* Initialize IDMAC DMA channels */
        mask = 0;
        for (i = 0; i < nchan; i++) {
                if (i > 0 && ((i * GTIDMAC_I_BITS) & 31 /*bit*/) == 0) {
                        bus_space_write_4(sc->sc_iot, sc->sc_ioh,
                            GTIDMAC_IMR(i - 1), mask);
                        mask = 0;
                }

                if (bus_dmamap_create(sc->sc_dmat, GTIDMAC_MAXXFER,
                    GTIDMAC_NSEGS, GTIDMAC_MAXXFER, 0, BUS_DMA_NOWAIT,
                    &sc->sc_cdesc[i].chan_in)) {
                        aprint_error_dev(sc->sc_dev,
                            "bus_dmamap_create failed: chan%d in\n", i);
                        goto fail4;
                }
                if (bus_dmamap_create(sc->sc_dmat, GTIDMAC_MAXXFER,
                    GTIDMAC_NSEGS, GTIDMAC_MAXXFER, 0, BUS_DMA_NOWAIT,
                    &sc->sc_cdesc[i].chan_out)) {
                        aprint_error_dev(sc->sc_dev,
                            "bus_dmamap_create failed: chan%d out\n", i);
                        bus_dmamap_destroy(sc->sc_dmat,
                            sc->sc_cdesc[i].chan_in);
                        goto fail4;
                }
                sc->sc_cdesc[i].chan_totalcnt = 0;
                sc->sc_cdesc[i].chan_running = NULL;

                /* Ignore bits overflow.  The mask is 32bit. */
                mask |= GTIDMAC_I(i,
                    GTIDMAC_I_COMP      |
                    GTIDMAC_I_ADDRMISS  |
                    GTIDMAC_I_ACCPROT   |
                    GTIDMAC_I_WRPROT    |
                    GTIDMAC_I_OWN);

                /* 8bits/channel * 4channels => 32bit */
                if ((i & 0x3) == 0x3) {
                        bus_space_write_4(sc->sc_iot, sc->sc_ioh,
                            GTIDMAC_IMR(i), mask);
                        mask = 0;
                }
        }

        return 0;

fail4:
        for (; i-- > 0;) {
                bus_dmamap_destroy(sc->sc_dmat, sc->sc_cdesc[i].chan_in);
                bus_dmamap_destroy(sc->sc_dmat, sc->sc_cdesc[i].chan_out);
        }
        bus_dmamap_unload(sc->sc_dmat, sc->sc_dmap);
fail3:
        bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmap);
fail2:
        bus_dmamem_unmap(sc->sc_dmat, sc->sc_dbuf,
            sizeof(struct gtidmac_desc) * GTIDMAC_NDESC);
fail1:
        bus_dmamem_free(sc->sc_dmat, &segs, 1);
fail0:
        return -1;
}

static int
mvxore_buffer_setup(struct gtidmac_softc *sc)
{
        bus_dma_segment_t segs;
        struct gtidmac_dma_desc *dd;
        uint32_t mask;
        int nchan, nsegs, i, j;

        nchan = sc->sc_mvxore_nchan;

        if (bus_dmamem_alloc(sc->sc_dmat,
            sizeof(struct mvxore_desc) * MVXORE_NDESC * nchan,
            PAGE_SIZE, 0, &segs, 1, &nsegs, BUS_DMA_NOWAIT)) {
                aprint_error_dev(sc->sc_dev,
                    "bus_dmamem_alloc failed: xore descriptor buffer\n");
                goto fail0;
        }
        if (bus_dmamem_map(sc->sc_dmat, &segs, 1,
            sizeof(struct mvxore_desc) * MVXORE_NDESC * nchan,
            (void **)&sc->sc_dbuf_xore, BUS_DMA_NOWAIT)) {
                aprint_error_dev(sc->sc_dev,
                    "bus_dmamem_map failed: xore descriptor buffer\n");
                goto fail1;
        }
        if (bus_dmamap_create(sc->sc_dmat,
            sizeof(struct mvxore_desc) * MVXORE_NDESC * nchan, 1,
            sizeof(struct mvxore_desc) * MVXORE_NDESC * nchan, 0,
            BUS_DMA_NOWAIT, &sc->sc_dmap_xore)) {
                aprint_error_dev(sc->sc_dev,
                    "bus_dmamap_create failed: xore descriptor buffer\n");
                goto fail2;
        }
        if (bus_dmamap_load(sc->sc_dmat, sc->sc_dmap_xore, sc->sc_dbuf_xore,
            sizeof(struct mvxore_desc) * MVXORE_NDESC * nchan,
            NULL, BUS_DMA_NOWAIT)) {
                aprint_error_dev(sc->sc_dev,
                    "bus_dmamap_load failed: xore descriptor buffer\n");
                goto fail3;
        }
        SLIST_INIT(&sc->sc_dlist_xore);
        for (i = 0; i < MVXORE_NDESC * nchan; i++) {
                dd =
                    &sc->sc_dd_buffer[i + GTIDMAC_NDESC * sc->sc_gtidmac_nchan];
                dd->dd_index = i;
                dd->dd_xore_vaddr = &sc->sc_dbuf_xore[i];
                dd->dd_paddr = sc->sc_dmap_xore->dm_segs[0].ds_addr +
                    (sizeof(struct mvxore_desc) * i);
                SLIST_INSERT_HEAD(&sc->sc_dlist_xore, dd, dd_next);
        }

        /* Initialize XORE DMA channels */
        mask = 0;
        for (i = 0; i < nchan; i++) {
                for (j = 0; j < MVXORE_NSRC; j++) {
                        if (bus_dmamap_create(sc->sc_dmat,
                            MVXORE_MAXXFER, MVXORE_NSEGS,
                            MVXORE_MAXXFER, 0, BUS_DMA_NOWAIT,
                            &sc->sc_cdesc_xore[i].chan_in[j])) {
                                aprint_error_dev(sc->sc_dev,
                                    "bus_dmamap_create failed:"
                                    " xore chan%d in[%d]\n", i, j);
                                goto fail4;
                        }
                }
                if (bus_dmamap_create(sc->sc_dmat, MVXORE_MAXXFER,
                    MVXORE_NSEGS, MVXORE_MAXXFER, 0,
                    BUS_DMA_NOWAIT, &sc->sc_cdesc_xore[i].chan_out)) {
                        aprint_error_dev(sc->sc_dev,
                            "bus_dmamap_create failed: chan%d out\n", i);
                        goto fail5;
                }
                sc->sc_cdesc_xore[i].chan_totalcnt = 0;
                sc->sc_cdesc_xore[i].chan_running = NULL;

                mask |= MVXORE_I(i,
                    MVXORE_I_EOC        |
                    MVXORE_I_ADDRDECODE |
                    MVXORE_I_ACCPROT    |
                    MVXORE_I_WRPROT     |
                    MVXORE_I_OWN        |
                    MVXORE_I_INTPARITY  |
                    MVXORE_I_XBAR);

                /* 16bits/channel * 2channels => 32bit */
                if (i & 0x1) {
                        bus_space_write_4(sc->sc_iot, sc->sc_ioh,
                            MVXORE_XEIMR(sc, i >> 1), mask);
                        mask = 0;
                }
        }

        return 0;

        for (; i-- > 0;) {
                bus_dmamap_destroy(sc->sc_dmat, sc->sc_cdesc_xore[i].chan_out);

fail5:
                j = MVXORE_NSRC;
fail4:
                for (; j-- > 0;)
                        bus_dmamap_destroy(sc->sc_dmat,
                            sc->sc_cdesc_xore[i].chan_in[j]);
        }
        bus_dmamap_unload(sc->sc_dmat, sc->sc_dmap_xore);
fail3:
        bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmap_xore);
fail2:
        bus_dmamem_unmap(sc->sc_dmat, sc->sc_dbuf_xore,
            sizeof(struct mvxore_desc) * MVXORE_NDESC);
fail1:
        bus_dmamem_free(sc->sc_dmat, &segs, 1);
fail0:
        return -1;
}

#ifdef GTIDMAC_DEBUG
static void
gtidmac_dump_idmacreg(struct gtidmac_softc *sc, int chan)
{
        uint32_t val;
        char buf[256];

        printf("IDMAC Registers\n");

        val = bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CIDMABCR(chan));
        snprintb(buf, sizeof(buf), "\177\020b\037Own\0b\036BCLeft\0", val);
        printf("  Byte Count                 : %s\n", buf);
        printf("    ByteCnt                  :   0x%06x\n",
            val & GTIDMAC_CIDMABCR_BYTECNT_MASK);
        printf("  Source Address             : 0x%08x\n",
            bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CIDMASAR(chan)));
        printf("  Destination Address        : 0x%08x\n",
            bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CIDMADAR(chan)));
        printf("  Next Descriptor Pointer    : 0x%08x\n",
            bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CNDPR(chan)));
        printf("  Current Descriptor Pointer : 0x%08x\n",
            bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCDPR(chan)));

        val = bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCLR(chan));
        snprintb(buf, sizeof(buf),
            "\177\020b\024Abr\0b\021CDEn\0b\016ChanAct\0b\015FetchND\0"
            "b\014ChanEn\0b\012IntMode\0b\005DestHold\0b\003SrcHold\0",
            val);
        printf("  Channel Control (Low)      : %s\n", buf);
        printf("    SrcBurstLimit            : %s Bytes\n",
          (val & GTIDMAC_CCLR_SBL_MASK) == GTIDMAC_CCLR_SBL_128B ? "128" :
            (val & GTIDMAC_CCLR_SBL_MASK) == GTIDMAC_CCLR_SBL_64B ? "64" :
            (val & GTIDMAC_CCLR_SBL_MASK) == GTIDMAC_CCLR_SBL_32B ? "32" :
            (val & GTIDMAC_CCLR_SBL_MASK) == GTIDMAC_CCLR_SBL_16B ? "16" :
            (val & GTIDMAC_CCLR_SBL_MASK) == GTIDMAC_CCLR_SBL_8B ? "8" :
            "unknown");
        printf("    DstBurstLimit            : %s Bytes\n",
          (val & GTIDMAC_CCLR_DBL_MASK) == GTIDMAC_CCLR_DBL_128B ? "128" :
            (val & GTIDMAC_CCLR_DBL_MASK) == GTIDMAC_CCLR_DBL_64B ? "64" :
            (val & GTIDMAC_CCLR_DBL_MASK) == GTIDMAC_CCLR_DBL_32B ? "32" :
            (val & GTIDMAC_CCLR_DBL_MASK) == GTIDMAC_CCLR_DBL_16B ? "16" :
            (val & GTIDMAC_CCLR_DBL_MASK) == GTIDMAC_CCLR_DBL_8B ? "8" :
            "unknown");
        printf("    ChainMode                : %sChained\n",
            val & GTIDMAC_CCLR_CHAINMODE_NC ? "Non-" : "");
        printf("    TransferMode             : %s\n",
            val & GTIDMAC_CCLR_TRANSFERMODE_B ? "Block" : "Demand");
        printf("    DescMode                 : %s\n",
            val & GTIDMAC_CCLR_DESCMODE_16M ? "16M" : "64k");
        val = bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCHR(chan));
        snprintb(buf, sizeof(buf),
            "\177\020b\001DescByteSwap\0b\000Endianness\0", val);
        printf("  Channel Control (High)     : %s\n", buf);
}

static void
gtidmac_dump_idmacdesc(struct gtidmac_softc *sc, struct gtidmac_dma_desc *dd,
                       uint32_t mode, int post)
{
        struct gtidmac_desc *desc;
        int i;
        char buf[256];

        printf("IDMAC Descriptor\n");

        i = 0;
        while (1 /*CONSTCOND*/) {
                if (post)
                        bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap,
                            dd->dd_index * sizeof(*desc), sizeof(*desc),
                            BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);

                desc = dd->dd_idmac_vaddr;

                printf("%d (0x%lx)\n", i, dd->dd_paddr);
                if (mode & GTIDMAC_CCLR_DESCMODE_16M) {
                        snprintb(buf, sizeof(buf),
                            "\177\020b\037Own\0b\036BCLeft\0",
                            desc->bc.mode16m.bcnt);
                        printf("  Byte Count              : %s\n", buf);
                        printf("    ByteCount             :   0x%06x\n",
                            desc->bc.mode16m.bcnt &
                            GTIDMAC_CIDMABCR_BYTECNT_MASK);
                } else {
                        printf("  Byte Count              :     0x%04x\n",
                            desc->bc.mode64k.bcnt);
                        printf("  Remind Byte Count       :     0x%04x\n",
                            desc->bc.mode64k.rbc);
                }
                printf("  Source Address          : 0x%08x\n", desc->srcaddr);
                printf("  Destination Address     : 0x%08x\n", desc->dstaddr);
                printf("  Next Descriptor Pointer : 0x%08x\n", desc->nextdp);

                if (desc->nextdp == (uint32_t)NULL)
                        break;

                if (!post)
                        bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap,
                            dd->dd_index * sizeof(*desc), sizeof(*desc),
                            BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

                i++;
                dd = SLIST_NEXT(dd, dd_next);
        }
        if (!post)
                bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap,
                    dd->dd_index * sizeof(*desc), sizeof(*desc),
                    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
}

static void
gtidmac_dump_xorereg(struct gtidmac_softc *sc, int chan)
{
        uint32_t val, opmode;
        char buf[64];

        printf("XORE Registers\n");

        val = bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXCR(sc, chan));
        snprintb(buf, sizeof(buf),
            "\177\020"
            "b\017RegAccProtect\0b\016DesSwp\0b\015DwrReqSwp\0b\014DrdResSwp\0",
            val);
        printf(" Configuration    : %s\n", buf);
        opmode = val & MVXORE_XEXCR_OM_MASK;
        printf("    OperationMode : %s operation\n",
          opmode == MVXORE_XEXCR_OM_XOR ? "XOR calculate" :
          opmode == MVXORE_XEXCR_OM_CRC32 ? "CRC-32 calculate" :
          opmode == MVXORE_XEXCR_OM_DMA ? "DMA" :
          opmode == MVXORE_XEXCR_OM_ECC ? "ECC cleanup" :
          opmode == MVXORE_XEXCR_OM_MEMINIT ? "Memory Initialization" :
          "unknown");
        printf("    SrcBurstLimit : %s Bytes\n",
          (val & MVXORE_XEXCR_SBL_MASK) == MVXORE_XEXCR_SBL_128B ? "128" :
            (val & MVXORE_XEXCR_SBL_MASK) == MVXORE_XEXCR_SBL_64B ? "64" :
            (val & MVXORE_XEXCR_SBL_MASK) == MVXORE_XEXCR_SBL_32B ? "32" :
            "unknown");
        printf("    DstBurstLimit : %s Bytes\n",
          (val & MVXORE_XEXCR_SBL_MASK) == MVXORE_XEXCR_SBL_128B ? "128" :
            (val & MVXORE_XEXCR_SBL_MASK) == MVXORE_XEXCR_SBL_64B ? "64" :
            (val & MVXORE_XEXCR_SBL_MASK) == MVXORE_XEXCR_SBL_32B ? "32" :
            "unknown");
        val =
            bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXACTR(sc, chan));
        printf("  Activation      : 0x%08x\n", val);
        val &= MVXORE_XEXACTR_XESTATUS_MASK;
        printf("    XEstatus      : %s\n",
            val == MVXORE_XEXACTR_XESTATUS_NA ? "Channel not active" :
            val == MVXORE_XEXACTR_XESTATUS_ACT ? "Channel active" :
            val == MVXORE_XEXACTR_XESTATUS_P ? "Channel paused" : "???");

        if (opmode == MVXORE_XEXCR_OM_XOR ||
            opmode == MVXORE_XEXCR_OM_CRC32 ||
            opmode == MVXORE_XEXCR_OM_DMA) {
                printf("  NextDescPtr     : 0x%08x\n",
                    bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                    MVXORE_XEXNDPR(sc, chan)));
                printf("  CurrentDescPtr  : 0x%08x\n",
                    bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                    MVXORE_XEXCDPR(sc, chan)));
        }
        printf("  ByteCnt         : 0x%08x\n",
            bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXBCR(sc, chan)));

        if (opmode == MVXORE_XEXCR_OM_ECC ||
            opmode == MVXORE_XEXCR_OM_MEMINIT) {
                printf("  DstPtr          : 0x%08x\n",
                    bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                    MVXORE_XEXDPR(sc, chan)));
                printf("  BlockSize       : 0x%08x\n",
                    bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                    MVXORE_XEXBSR(sc, chan)));

                if (opmode == MVXORE_XEXCR_OM_ECC) {
                        val = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                            MVXORE_XETMCR);
                        if (val & MVXORE_XETMCR_TIMEREN) {
                                val >>= MVXORE_XETMCR_SECTIONSIZECTRL_SHIFT;
                                val &= MVXORE_XETMCR_SECTIONSIZECTRL_MASK;
                                printf("  SectionSizeCtrl : 0x%08x\n", 2 ^ val);
                                printf("  TimerInitVal    : 0x%08x\n",
                                    bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                                    MVXORE_XETMIVR));
                                printf("  TimerCrntVal    : 0x%08x\n",
                                    bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                                    MVXORE_XETMCVR));
                        }
                } else  /* MVXORE_XEXCR_OM_MEMINIT */
                        printf("  InitVal         : 0x%08x%08x\n",
                            bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                            MVXORE_XEIVRH),
                            bus_space_read_4(sc->sc_iot, sc->sc_ioh,
                            MVXORE_XEIVRL));
        }
}

static void
gtidmac_dump_xoredesc(struct gtidmac_softc *sc, struct gtidmac_dma_desc *dd,
                      uint32_t mode, int post)
{
        struct mvxore_desc *desc;
        int i, j;
        char buf[256];

        printf("XORE Descriptor\n");

        mode &= MVXORE_XEXCR_OM_MASK;

        i = 0;
        while (1 /*CONSTCOND*/) {
                if (post)
                        bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap_xore,
                            dd->dd_index * sizeof(*desc), sizeof(*desc),
                            BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);

                desc = dd->dd_xore_vaddr;

                printf("%d (0x%lx)\n", i, dd->dd_paddr);

                snprintb(buf, sizeof(buf), "\177\020b\037Own\0b\036Success\0",
                    desc->stat);
                printf("  Status                  : %s\n", buf);
                if (desc->cmd & MVXORE_DESC_CMD_CRCLAST && post)
                        printf("  CRC-32 Result           : 0x%08x\n",
                            desc->result);
                snprintb(buf, sizeof(buf),
                    "\177\020b\037EODIntEn\0b\036CRCLast\0"
                    "b\007Src7Cmd\0b\006Src6Cmd\0b\005Src5Cmd\0b\004Src4Cmd\0"
                    "b\003Src3Cmd\0b\002Src2Cmd\0b\001Src1Cmd\0b\000Src0Cmd\0",
                    desc->cmd);
                printf("  Command                 : %s\n", buf);
                printf("  Next Descriptor Address : 0x%08x\n", desc->nextda);
                printf("  Byte Count              :   0x%06x\n", desc->bcnt);
                printf("  Destination Address     : 0x%08x\n", desc->dstaddr);
                if (mode == MVXORE_XEXCR_OM_XOR) {
                        for (j = 0; j < MVXORE_NSRC; j++)
                                if (desc->cmd & MVXORE_DESC_CMD_SRCCMD(j))
                                        printf("  Source Address#%d        :"
                                            " 0x%08x\n", j, desc->srcaddr[j]);
                } else
                        printf("  Source Address          : 0x%08x\n",
                            desc->srcaddr[0]);

                if (desc->nextda == (uint32_t)NULL)
                        break;

                if (!post)
                        bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap_xore,
                            dd->dd_index * sizeof(*desc), sizeof(*desc),
                            BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

                i++;
                dd = SLIST_NEXT(dd, dd_next);
        }
        if (!post)
                bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap_xore,
                    dd->dd_index * sizeof(*desc), sizeof(*desc),
                    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
}
#endif