// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Loongson-2 Chain Multi-Channel DMA Controller driver
 *
 * Copyright (C) 2024-2026 Loongson Technology Corporation Limited
 */

#include <linux/acpi.h>
#include <linux/acpi_dma.h>
#include <linux/bitfield.h>
#include <linux/clk.h>
#include <linux/dma-mapping.h>
#include <linux/dmapool.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_dma.h>
#include <linux/platform_device.h>
#include <linux/slab.h>

#include "../dmaengine.h"
#include "../virt-dma.h"

#define LOONGSON2_CMCDMA_ISR            0x0     /* DMA Interrupt Status Register */
#define LOONGSON2_CMCDMA_IFCR           0x4     /* DMA Interrupt Flag Clear Register */
#define LOONGSON2_CMCDMA_CCR            0x8     /* DMA Channel Configuration Register */
#define LOONGSON2_CMCDMA_CNDTR          0xc     /* DMA Channel Transmit Count Register */
#define LOONGSON2_CMCDMA_CPAR           0x10    /* DMA Channel Peripheral Address Register */
#define LOONGSON2_CMCDMA_CMAR           0x14    /* DMA Channel Memory Address Register */

/* Bitfields of DMA interrupt status register */
#define LOONGSON2_CMCDMA_TCI            BIT(1) /* Transfer Complete Interrupt */
#define LOONGSON2_CMCDMA_HTI            BIT(2) /* Half Transfer Interrupt */
#define LOONGSON2_CMCDMA_TEI            BIT(3) /* Transfer Error Interrupt */

#define LOONGSON2_CMCDMA_MASKI          \
        (LOONGSON2_CMCDMA_TCI | LOONGSON2_CMCDMA_HTI | LOONGSON2_CMCDMA_TEI)

/* Bitfields of DMA channel x Configuration Register */
#define LOONGSON2_CMCDMA_CCR_EN         BIT(0) /* Stream Enable */
#define LOONGSON2_CMCDMA_CCR_TCIE       BIT(1) /* Transfer Complete Interrupt Enable */
#define LOONGSON2_CMCDMA_CCR_HTIE       BIT(2) /* Half Transfer Complete Interrupt Enable */
#define LOONGSON2_CMCDMA_CCR_TEIE       BIT(3) /* Transfer Error Interrupt Enable */
#define LOONGSON2_CMCDMA_CCR_DIR        BIT(4) /* Data Transfer Direction */
#define LOONGSON2_CMCDMA_CCR_CIRC       BIT(5) /* Circular mode */
#define LOONGSON2_CMCDMA_CCR_PINC       BIT(6) /* Peripheral increment mode */
#define LOONGSON2_CMCDMA_CCR_MINC       BIT(7) /* Memory increment mode */
#define LOONGSON2_CMCDMA_CCR_PSIZE_MASK GENMASK(9, 8)
#define LOONGSON2_CMCDMA_CCR_MSIZE_MASK GENMASK(11, 10)
#define LOONGSON2_CMCDMA_CCR_PL_MASK    GENMASK(13, 12)
#define LOONGSON2_CMCDMA_CCR_M2M        BIT(14)

#define LOONGSON2_CMCDMA_CCR_CFG_MASK   \
        (LOONGSON2_CMCDMA_CCR_PINC | LOONGSON2_CMCDMA_CCR_MINC | LOONGSON2_CMCDMA_CCR_PL_MASK)

#define LOONGSON2_CMCDMA_CCR_IRQ_MASK   \
        (LOONGSON2_CMCDMA_CCR_TCIE | LOONGSON2_CMCDMA_CCR_HTIE | LOONGSON2_CMCDMA_CCR_TEIE)

#define LOONGSON2_CMCDMA_STREAM_MASK    \
        (LOONGSON2_CMCDMA_CCR_CFG_MASK | LOONGSON2_CMCDMA_CCR_IRQ_MASK)

#define LOONGSON2_CMCDMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
                                         BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
                                         BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))

#define LOONSON2_CMCDMA_MAX_DATA_ITEMS  SZ_64K

struct loongson2_cmc_dma_chan_reg {
        u32 ccr;
        u32 cndtr;
        u32 cpar;
        u32 cmar;
};

struct loongson2_cmc_dma_sg_req {
        u32 len;
        struct loongson2_cmc_dma_chan_reg chan_reg;
};

struct loongson2_cmc_dma_desc {
        struct virt_dma_desc vdesc;
        bool cyclic;
        u32 num_sgs;
        struct loongson2_cmc_dma_sg_req sg_req[] __counted_by(num_sgs);
};

struct loongson2_cmc_dma_chan {
        struct virt_dma_chan vchan;
        struct dma_slave_config dma_sconfig;
        struct loongson2_cmc_dma_desc *desc;
        u32 id;
        u32 irq;
        u32 next_sg;
        struct loongson2_cmc_dma_chan_reg chan_reg;
};

struct loongson2_cmc_dma_dev {
        struct dma_device ddev;
        struct clk *dma_clk;
        void __iomem *base;
        u32 nr_channels;
        u32 chan_reg_offset;
        struct loongson2_cmc_dma_chan chan[] __counted_by(nr_channels);
};

struct loongson2_cmc_dma_config {
        u32 max_channels;
        u32 chan_reg_offset;
};

static const struct loongson2_cmc_dma_config ls2k0300_cmc_dma_config = {
        .max_channels = 8,
        .chan_reg_offset = 0x14,
};

static const struct loongson2_cmc_dma_config ls2k3000_cmc_dma_config = {
        .max_channels = 4,
        .chan_reg_offset = 0x18,
};

static struct loongson2_cmc_dma_dev *lmdma_get_dev(struct loongson2_cmc_dma_chan *lchan)
{
        return container_of(lchan->vchan.chan.device, struct loongson2_cmc_dma_dev, ddev);
}

static struct loongson2_cmc_dma_chan *to_lmdma_chan(struct dma_chan *chan)
{
        return container_of(chan, struct loongson2_cmc_dma_chan, vchan.chan);
}

static struct loongson2_cmc_dma_desc *to_lmdma_desc(struct virt_dma_desc *vdesc)
{
        return container_of(vdesc, struct loongson2_cmc_dma_desc, vdesc);
}

static struct device *chan2dev(struct loongson2_cmc_dma_chan *lchan)
{
        return &lchan->vchan.chan.dev->device;
}

static u32 loongson2_cmc_dma_read(struct loongson2_cmc_dma_dev *lddev, u32 reg, u32 id)
{
        return readl(lddev->base + (reg + lddev->chan_reg_offset * id));
}

static void loongson2_cmc_dma_write(struct loongson2_cmc_dma_dev *lddev, u32 reg, u32 id, u32 val)
{
        writel(val, lddev->base + (reg + lddev->chan_reg_offset * id));
}

static int loongson2_cmc_dma_get_width(enum dma_slave_buswidth width)
{
        switch (width) {
        case DMA_SLAVE_BUSWIDTH_1_BYTE:
        case DMA_SLAVE_BUSWIDTH_2_BYTES:
        case DMA_SLAVE_BUSWIDTH_4_BYTES:
                return ffs(width) - 1;
        default:
                return -EINVAL;
        }
}

static int loongson2_cmc_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *config)
{
        struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);

        memcpy(&lchan->dma_sconfig, config, sizeof(*config));

        return 0;
}

static void loongson2_cmc_dma_irq_clear(struct loongson2_cmc_dma_chan *lchan, u32 flags)
{
        struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
        u32 ifcr;

        ifcr = flags << (4 * lchan->id);
        loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_IFCR, 0, ifcr);
}

static void loongson2_cmc_dma_stop(struct loongson2_cmc_dma_chan *lchan)
{
        struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
        u32 ccr;

        ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id);
        ccr &= ~(LOONGSON2_CMCDMA_CCR_IRQ_MASK | LOONGSON2_CMCDMA_CCR_EN);
        loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, ccr);

        loongson2_cmc_dma_irq_clear(lchan, LOONGSON2_CMCDMA_MASKI);
}

static int loongson2_cmc_dma_terminate_all(struct dma_chan *chan)
{
        struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);

        LIST_HEAD(head);

        scoped_guard(spinlock_irqsave, &lchan->vchan.lock) {
                if (lchan->desc) {
                        vchan_terminate_vdesc(&lchan->desc->vdesc);
                        loongson2_cmc_dma_stop(lchan);
                        lchan->desc = NULL;
                }
                vchan_get_all_descriptors(&lchan->vchan, &head);
        }

        vchan_dma_desc_free_list(&lchan->vchan, &head);

        return 0;
}

static void loongson2_cmc_dma_synchronize(struct dma_chan *chan)
{
        struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);

        vchan_synchronize(&lchan->vchan);
}

static void loongson2_cmc_dma_start_transfer(struct loongson2_cmc_dma_chan *lchan)
{
        struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
        struct loongson2_cmc_dma_sg_req *sg_req;
        struct loongson2_cmc_dma_chan_reg *reg;
        struct virt_dma_desc *vdesc;

        loongson2_cmc_dma_stop(lchan);

        if (!lchan->desc) {
                vdesc = vchan_next_desc(&lchan->vchan);
                if (!vdesc)
                        return;

                list_del(&vdesc->node);
                lchan->desc = to_lmdma_desc(vdesc);
                lchan->next_sg = 0;
        }

        if (lchan->next_sg == lchan->desc->num_sgs)
                lchan->next_sg = 0;

        sg_req = &lchan->desc->sg_req[lchan->next_sg];
        reg = &sg_req->chan_reg;

        loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, reg->ccr);
        loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CNDTR, lchan->id, reg->cndtr);
        loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CPAR, lchan->id, reg->cpar);
        loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CMAR, lchan->id, reg->cmar);

        lchan->next_sg++;

        /* Start DMA */
        reg->ccr |= LOONGSON2_CMCDMA_CCR_EN;
        loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, reg->ccr);
}

static void loongson2_cmc_dma_configure_next_sg(struct loongson2_cmc_dma_chan *lchan)
{
        struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
        struct loongson2_cmc_dma_sg_req *sg_req;
        u32 ccr, id = lchan->id;

        if (lchan->next_sg == lchan->desc->num_sgs)
                lchan->next_sg = 0;

        /* Stop to update mem addr */
        ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, id);
        ccr &= ~LOONGSON2_CMCDMA_CCR_EN;
        loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, id, ccr);

        sg_req = &lchan->desc->sg_req[lchan->next_sg];
        loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CMAR, id, sg_req->chan_reg.cmar);

        /* Start transition */
        ccr |= LOONGSON2_CMCDMA_CCR_EN;
        loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, id, ccr);
}

static void loongson2_cmc_dma_handle_chan_done(struct loongson2_cmc_dma_chan *lchan)
{
        if (!lchan->desc)
                return;

        if (lchan->desc->cyclic) {
                vchan_cyclic_callback(&lchan->desc->vdesc);
                /* LOONGSON2_CMCDMA_CCR_CIRC mode don't need update register */
                if (lchan->desc->num_sgs == 1)
                        return;
                loongson2_cmc_dma_configure_next_sg(lchan);
                lchan->next_sg++;
        } else {
                if (lchan->next_sg == lchan->desc->num_sgs) {
                        vchan_cookie_complete(&lchan->desc->vdesc);
                        lchan->desc = NULL;
                }
                loongson2_cmc_dma_start_transfer(lchan);
        }
}

static irqreturn_t loongson2_cmc_dma_chan_irq(int irq, void *devid)
{
        struct loongson2_cmc_dma_chan *lchan = devid;
        struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
        struct device *dev = chan2dev(lchan);
        u32 ists, status, ccr;

        scoped_guard(spinlock, &lchan->vchan.lock) {
                ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id);
                ists = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_ISR, 0);
                status = (ists >> (4 * lchan->id)) & LOONGSON2_CMCDMA_MASKI;

                loongson2_cmc_dma_irq_clear(lchan, status);

                if (status & LOONGSON2_CMCDMA_TCI) {
                        loongson2_cmc_dma_handle_chan_done(lchan);
                        status &= ~LOONGSON2_CMCDMA_TCI;
                }

                if (status & LOONGSON2_CMCDMA_HTI)
                        status &= ~LOONGSON2_CMCDMA_HTI;

                if (status & LOONGSON2_CMCDMA_TEI) {
                        dev_err(dev, "DMA Transform Error.\n");
                        if (!(ccr & LOONGSON2_CMCDMA_CCR_EN))
                                dev_err(dev, "Channel disabled by HW.\n");
                }
        }

        return IRQ_HANDLED;
}

static void loongson2_cmc_dma_issue_pending(struct dma_chan *chan)
{
        struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);

        guard(spinlock_irqsave)(&lchan->vchan.lock);

        if (vchan_issue_pending(&lchan->vchan) && !lchan->desc) {
                dev_dbg(chan2dev(lchan), "vchan %pK: issued\n", &lchan->vchan);
                loongson2_cmc_dma_start_transfer(lchan);
        }
}

static int loongson2_cmc_dma_set_xfer_param(struct loongson2_cmc_dma_chan *lchan,
                                            enum dma_transfer_direction direction,
                                            enum dma_slave_buswidth *buswidth, u32 buf_len)
{
        struct dma_slave_config sconfig = lchan->dma_sconfig;
        struct device *dev = chan2dev(lchan);
        int dev_width;
        u32 ccr;

        switch (direction) {
        case DMA_MEM_TO_DEV:
                dev_width = loongson2_cmc_dma_get_width(sconfig.dst_addr_width);
                if (dev_width < 0) {
                        dev_err(dev, "DMA_MEM_TO_DEV bus width not supported\n");
                        return dev_width;
                }
                lchan->chan_reg.cpar = sconfig.dst_addr;
                ccr = LOONGSON2_CMCDMA_CCR_DIR;
                *buswidth = sconfig.dst_addr_width;
                break;
        case DMA_DEV_TO_MEM:
                dev_width = loongson2_cmc_dma_get_width(sconfig.src_addr_width);
                if (dev_width < 0) {
                        dev_err(dev, "DMA_DEV_TO_MEM bus width not supported\n");
                        return dev_width;
                }
                lchan->chan_reg.cpar = sconfig.src_addr;
                ccr = LOONGSON2_CMCDMA_CCR_MINC;
                *buswidth = sconfig.src_addr_width;
                break;
        default:
                return -EINVAL;
        }

        ccr |= FIELD_PREP(LOONGSON2_CMCDMA_CCR_PSIZE_MASK, dev_width) |
               FIELD_PREP(LOONGSON2_CMCDMA_CCR_MSIZE_MASK, dev_width);

        /* Set DMA control register */
        lchan->chan_reg.ccr &= ~(LOONGSON2_CMCDMA_CCR_PSIZE_MASK | LOONGSON2_CMCDMA_CCR_MSIZE_MASK);
        lchan->chan_reg.ccr |= ccr;

        return 0;
}

static struct dma_async_tx_descriptor *
loongson2_cmc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, u32 sg_len,
                                enum dma_transfer_direction direction,
                                unsigned long flags, void *context)
{
        struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
        struct loongson2_cmc_dma_desc *desc;
        enum dma_slave_buswidth buswidth;
        struct scatterlist *sg;
        u32 num_items, i;
        int ret;

        desc = kzalloc_flex(*desc, sg_req, sg_len, GFP_NOWAIT);
        if (!desc)
                return ERR_PTR(-ENOMEM);

        for_each_sg(sgl, sg, sg_len, i) {
                ret = loongson2_cmc_dma_set_xfer_param(lchan, direction, &buswidth, sg_dma_len(sg));
                if (ret)
                        return ERR_PTR(ret);

                num_items = DIV_ROUND_UP(sg_dma_len(sg), buswidth);
                if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) {
                        dev_err(chan2dev(lchan), "Number of items not supported\n");
                        kfree(desc);
                        return ERR_PTR(-EINVAL);
                }

                desc->sg_req[i].len = sg_dma_len(sg);
                desc->sg_req[i].chan_reg.ccr = lchan->chan_reg.ccr;
                desc->sg_req[i].chan_reg.cpar = lchan->chan_reg.cpar;
                desc->sg_req[i].chan_reg.cmar = sg_dma_address(sg);
                desc->sg_req[i].chan_reg.cndtr = num_items;
        }

        desc->num_sgs = sg_len;
        desc->cyclic = false;

        return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags);
}

static struct dma_async_tx_descriptor *
loongson2_cmc_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
                                  size_t period_len, enum dma_transfer_direction direction,
                                  unsigned long flags)
{
        struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
        struct loongson2_cmc_dma_desc *desc;
        enum dma_slave_buswidth buswidth;
        u32 num_periods, num_items, i;
        int ret;

        if (unlikely(buf_len % period_len))
                return ERR_PTR(-EINVAL);

        ret = loongson2_cmc_dma_set_xfer_param(lchan, direction, &buswidth, period_len);
        if (ret)
                return ERR_PTR(ret);

        num_items = DIV_ROUND_UP(period_len, buswidth);
        if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) {
                dev_err(chan2dev(lchan), "Number of items not supported\n");
                return ERR_PTR(-EINVAL);
        }

        /* Enable Circular mode */
        if (buf_len == period_len)
                lchan->chan_reg.ccr |= LOONGSON2_CMCDMA_CCR_CIRC;

        num_periods = DIV_ROUND_UP(buf_len, period_len);
        desc = kzalloc_flex(*desc, sg_req, num_periods, GFP_NOWAIT);
        if (!desc)
                return ERR_PTR(-ENOMEM);

        for (i = 0; i < num_periods; i++) {
                desc->sg_req[i].len = period_len;
                desc->sg_req[i].chan_reg.ccr = lchan->chan_reg.ccr;
                desc->sg_req[i].chan_reg.cpar = lchan->chan_reg.cpar;
                desc->sg_req[i].chan_reg.cmar = buf_addr;
                desc->sg_req[i].chan_reg.cndtr = num_items;
                buf_addr += period_len;
        }

        desc->num_sgs = num_periods;
        desc->cyclic = true;

        return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags);
}

static size_t loongson2_cmc_dma_desc_residue(struct loongson2_cmc_dma_chan *lchan,
                                             struct loongson2_cmc_dma_desc *desc, u32 next_sg)
{
        struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
        u32 residue, width, ndtr, ccr, i;

        ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id);
        width = FIELD_GET(LOONGSON2_CMCDMA_CCR_PSIZE_MASK, ccr);

        ndtr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CNDTR, lchan->id);
        residue = ndtr << width;

        if (lchan->desc->cyclic && next_sg == 0)
                return residue;

        for (i = next_sg; i < desc->num_sgs; i++)
                residue += desc->sg_req[i].len;

        return residue;
}

static enum dma_status loongson2_cmc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
                                                   struct dma_tx_state *state)
{
        struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
        struct virt_dma_desc *vdesc;
        enum dma_status status;

        status = dma_cookie_status(chan, cookie, state);
        if (status == DMA_COMPLETE || !state)
                return status;

        scoped_guard(spinlock_irqsave, &lchan->vchan.lock) {
                vdesc = vchan_find_desc(&lchan->vchan, cookie);
                if (lchan->desc && cookie == lchan->desc->vdesc.tx.cookie)
                        state->residue = loongson2_cmc_dma_desc_residue(lchan, lchan->desc,
                                                                        lchan->next_sg);
                else if (vdesc)
                        state->residue = loongson2_cmc_dma_desc_residue(lchan,
                                                                        to_lmdma_desc(vdesc), 0);
        }

        return status;
}

static void loongson2_cmc_dma_free_chan_resources(struct dma_chan *chan)
{
        vchan_free_chan_resources(to_virt_chan(chan));
}

static void loongson2_cmc_dma_desc_free(struct virt_dma_desc *vdesc)
{
        kfree(to_lmdma_desc(vdesc));
}

static bool loongson2_cmc_dma_acpi_filter(struct dma_chan *chan, void *param)
{
        struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan);
        struct acpi_dma_spec *dma_spec = param;

        memset(&lchan->chan_reg, 0, sizeof(struct loongson2_cmc_dma_chan_reg));
        lchan->chan_reg.ccr = dma_spec->chan_id & LOONGSON2_CMCDMA_STREAM_MASK;

        return true;
}

static int loongson2_cmc_dma_acpi_controller_register(struct loongson2_cmc_dma_dev *lddev)
{
        struct device *dev = lddev->ddev.dev;
        struct acpi_dma_filter_info *info;

        if (!is_acpi_node(dev_fwnode(dev)))
                return 0;

        info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
        if (!info)
                return -ENOMEM;

        dma_cap_zero(info->dma_cap);
        info->dma_cap = lddev->ddev.cap_mask;
        info->filter_fn = loongson2_cmc_dma_acpi_filter;

        return devm_acpi_dma_controller_register(dev, acpi_dma_simple_xlate, info);
}

static struct dma_chan *loongson2_cmc_dma_of_xlate(struct of_phandle_args *dma_spec,
                                                   struct of_dma *ofdma)
{
        struct loongson2_cmc_dma_dev *lddev = ofdma->of_dma_data;
        struct device *dev = lddev->ddev.dev;
        struct loongson2_cmc_dma_chan *lchan;
        struct dma_chan *chan;

        if (dma_spec->args_count < 2)
                return ERR_PTR(-EINVAL);

        if (dma_spec->args[0] >= lddev->nr_channels) {
                dev_err(dev, "Invalid channel id.\n");
                return ERR_PTR(-EINVAL);
        }

        lchan = &lddev->chan[dma_spec->args[0]];
        chan = dma_get_slave_channel(&lchan->vchan.chan);
        if (!chan) {
                dev_err(dev, "No more channels available.\n");
                return ERR_PTR(-EINVAL);
        }

        memset(&lchan->chan_reg, 0, sizeof(struct loongson2_cmc_dma_chan_reg));
        lchan->chan_reg.ccr = dma_spec->args[1] & LOONGSON2_CMCDMA_STREAM_MASK;

        return chan;
}

static int loongson2_cmc_dma_of_controller_register(struct loongson2_cmc_dma_dev *lddev)
{
        struct device *dev = lddev->ddev.dev;

        if (!is_of_node(dev_fwnode(dev)))
                return 0;

        return of_dma_controller_register(dev->of_node, loongson2_cmc_dma_of_xlate, lddev);
}

static int loongson2_cmc_dma_probe(struct platform_device *pdev)
{
        const struct loongson2_cmc_dma_config *config;
        struct loongson2_cmc_dma_chan *lchan;
        struct loongson2_cmc_dma_dev *lddev;
        struct device *dev = &pdev->dev;
        struct dma_device *ddev;
        u32 nr_chans, i;
        int ret;

        config = (const struct loongson2_cmc_dma_config *)device_get_match_data(dev);
        if (!config)
                return -EINVAL;

        ret = device_property_read_u32(dev, "dma-channels", &nr_chans);
        if (ret || nr_chans > config->max_channels) {
                dev_err(dev, "missing or invalid dma-channels property\n");
                nr_chans = config->max_channels;
        }

        lddev = devm_kzalloc(dev, struct_size(lddev, chan, nr_chans), GFP_KERNEL);
        if (!lddev)
                return -ENOMEM;

        lddev->base = devm_platform_ioremap_resource(pdev, 0);
        if (IS_ERR(lddev->base))
                return PTR_ERR(lddev->base);

        platform_set_drvdata(pdev, lddev);
        lddev->nr_channels = nr_chans;
        lddev->chan_reg_offset = config->chan_reg_offset;

        lddev->dma_clk = devm_clk_get_optional_enabled(dev, NULL);
        if (IS_ERR(lddev->dma_clk))
                return dev_err_probe(dev, PTR_ERR(lddev->dma_clk), "Failed to get dma clock\n");

        ddev = &lddev->ddev;
        ddev->dev = dev;

        dma_cap_zero(ddev->cap_mask);
        dma_cap_set(DMA_SLAVE, ddev->cap_mask);
        dma_cap_set(DMA_PRIVATE, ddev->cap_mask);
        dma_cap_set(DMA_CYCLIC, ddev->cap_mask);

        ddev->device_free_chan_resources = loongson2_cmc_dma_free_chan_resources;
        ddev->device_config = loongson2_cmc_dma_slave_config;
        ddev->device_prep_slave_sg = loongson2_cmc_dma_prep_slave_sg;
        ddev->device_prep_dma_cyclic = loongson2_cmc_dma_prep_dma_cyclic;
        ddev->device_issue_pending = loongson2_cmc_dma_issue_pending;
        ddev->device_synchronize = loongson2_cmc_dma_synchronize;
        ddev->device_tx_status = loongson2_cmc_dma_tx_status;
        ddev->device_terminate_all = loongson2_cmc_dma_terminate_all;

        ddev->max_sg_burst = LOONSON2_CMCDMA_MAX_DATA_ITEMS;
        ddev->src_addr_widths = LOONGSON2_CMCDMA_BUSWIDTHS;
        ddev->dst_addr_widths = LOONGSON2_CMCDMA_BUSWIDTHS;
        ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
        INIT_LIST_HEAD(&ddev->channels);

        for (i = 0; i < nr_chans; i++) {
                lchan = &lddev->chan[i];

                lchan->id = i;
                lchan->vchan.desc_free = loongson2_cmc_dma_desc_free;
                vchan_init(&lchan->vchan, ddev);
        }

        ret = dmaenginem_async_device_register(ddev);
        if (ret)
                return dev_err_probe(dev, ret, "Failed to register DMA engine device.\n");

        for (i = 0; i < nr_chans; i++) {
                lchan = &lddev->chan[i];

                lchan->irq = platform_get_irq(pdev, i);
                if (lchan->irq < 0)
                        return lchan->irq;

                ret = devm_request_irq(dev, lchan->irq, loongson2_cmc_dma_chan_irq, IRQF_SHARED,
                                       dev_name(chan2dev(lchan)), lchan);
                if (ret)
                        return ret;
        }

        ret = loongson2_cmc_dma_acpi_controller_register(lddev);
        if (ret)
                return dev_err_probe(dev, ret, "Failed to register dma controller with ACPI.\n");

        ret = loongson2_cmc_dma_of_controller_register(lddev);
        if (ret)
                return dev_err_probe(dev, ret, "Failed to register dma controller with FDT.\n");

        dev_info(dev, "Loongson-2 Multi-Channel DMA Controller registered successfully.\n");

        return 0;
}

static void loongson2_cmc_dma_remove(struct platform_device *pdev)
{
        of_dma_controller_free(pdev->dev.of_node);
}

static const struct of_device_id loongson2_cmc_dma_of_match[] = {
        { .compatible = "loongson,ls2k0300-dma", .data = &ls2k0300_cmc_dma_config },
        { .compatible = "loongson,ls2k3000-dma", .data = &ls2k3000_cmc_dma_config },
        { /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, loongson2_cmc_dma_of_match);

static const struct acpi_device_id loongson2_cmc_dma_acpi_match[] = {
        { "LOON0014", .driver_data = (kernel_ulong_t)&ls2k3000_cmc_dma_config },
        { /* sentinel */ }
};
MODULE_DEVICE_TABLE(acpi, loongson2_cmc_dma_acpi_match);

static struct platform_driver loongson2_cmc_dma_driver = {
        .driver = {
                .name = "loongson2-apb-cmc-dma",
                .of_match_table = loongson2_cmc_dma_of_match,
                .acpi_match_table = loongson2_cmc_dma_acpi_match,
        },
        .probe = loongson2_cmc_dma_probe,
        .remove = loongson2_cmc_dma_remove,
};
module_platform_driver(loongson2_cmc_dma_driver);

MODULE_DESCRIPTION("Loongson-2 Chain Multi-Channel DMA Controller driver");
MODULE_AUTHOR("Loongson Technology Corporation Limited");
MODULE_LICENSE("GPL");