#include <linux/bitfield.h>
#include <linux/cleanup.h>
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/interrupt.h>
#include <linux/iopoll.h>
#include <linux/irqchip/irq-renesas-rzv2h.h>
#include <linux/irqchip/irq-renesas-rzt2h.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_dma.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/reset.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include "../dmaengine.h"
#include "../virt-dma.h"
enum rz_dmac_prep_type {
RZ_DMAC_DESC_MEMCPY,
RZ_DMAC_DESC_SLAVE_SG,
RZ_DMAC_DESC_CYCLIC,
};
struct rz_lmdesc {
u32 header;
u32 sa;
u32 da;
u32 tb;
u32 chcfg;
u32 chitvl;
u32 chext;
u32 nxla;
};
struct rz_dmac_desc {
struct virt_dma_desc vd;
dma_addr_t src;
dma_addr_t dest;
size_t len;
struct list_head node;
enum dma_transfer_direction direction;
enum rz_dmac_prep_type type;
struct scatterlist *sg;
unsigned int sgcount;
struct rz_lmdesc *start_lmdesc;
};
#define to_rz_dmac_desc(d) container_of(d, struct rz_dmac_desc, vd)
enum rz_dmac_chan_status {
RZ_DMAC_CHAN_STATUS_PAUSED,
RZ_DMAC_CHAN_STATUS_CYCLIC,
RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL,
};
struct rz_dmac_chan {
struct virt_dma_chan vc;
void __iomem *ch_base;
void __iomem *ch_cmn_base;
unsigned int index;
struct rz_dmac_desc *desc;
int descs_allocated;
dma_addr_t src_per_address;
dma_addr_t dst_per_address;
unsigned long status;
u32 chcfg;
u32 chctrl;
int mid_rid;
int dmac_ack;
struct {
u32 nxla;
} pm_state;
struct list_head ld_free;
struct {
struct rz_lmdesc *base;
struct rz_lmdesc *head;
struct rz_lmdesc *tail;
dma_addr_t base_dma;
} lmdesc;
};
#define to_rz_dmac_chan(c) container_of(c, struct rz_dmac_chan, vc.chan)
struct rz_dmac_icu {
struct platform_device *pdev;
u8 dmac_index;
};
struct rz_dmac_info {
void (*icu_register_dma_req)(struct platform_device *icu_dev,
u8 dmac_index, u8 dmac_channel, u16 req_no);
void (*icu_register_dma_ack)(struct platform_device *icu_dev,
u8 dmac_index, u8 dmac_channel, u16 ack_no);
u16 default_dma_ack_no;
u16 default_dma_req_no;
};
struct rz_dmac {
struct dma_device engine;
struct rz_dmac_icu icu;
const struct rz_dmac_info *info;
struct device *dev;
struct reset_control *rstc;
void __iomem *base;
void __iomem *ext_base;
unsigned int n_channels;
struct rz_dmac_chan *channels;
DECLARE_BITMAP(modules, 1024);
};
#define to_rz_dmac(d) container_of(d, struct rz_dmac, engine)
#define CRTB 0x0020
#define CHSTAT 0x0024
#define CHCTRL 0x0028
#define CHCFG 0x002c
#define NXLA 0x0038
#define CRLA 0x003c
#define DCTRL 0x0000
#define EACH_CHANNEL_OFFSET 0x0040
#define CHANNEL_0_7_OFFSET 0x0000
#define CHANNEL_0_7_COMMON_BASE 0x0300
#define CHANNEL_8_15_OFFSET 0x0400
#define CHANNEL_8_15_COMMON_BASE 0x0700
#define CHSTAT_ER BIT(4)
#define CHSTAT_SUS BIT(3)
#define CHSTAT_EN BIT(0)
#define CHCTRL_CLRINTMSK BIT(17)
#define CHCTRL_CLRSUS BIT(9)
#define CHCTRL_SETSUS BIT(8)
#define CHCTRL_CLRTC BIT(6)
#define CHCTRL_CLREND BIT(5)
#define CHCTRL_CLRRQ BIT(4)
#define CHCTRL_SWRST BIT(3)
#define CHCTRL_STG BIT(2)
#define CHCTRL_CLREN BIT(1)
#define CHCTRL_SETEN BIT(0)
#define CHCTRL_DEFAULT (CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \
CHCTRL_CLRTC | CHCTRL_CLREND | \
CHCTRL_CLRRQ | CHCTRL_SWRST | \
CHCTRL_CLREN)
#define CHCFG_DMS BIT(31)
#define CHCFG_DEM BIT(24)
#define CHCFG_DAD BIT(21)
#define CHCFG_SAD BIT(20)
#define CHCFG_REQD BIT(3)
#define CHCFG_SEL(bits) ((bits) & 0x07)
#define CHCFG_MEM_COPY (0x80400008)
#define CHCFG_FILL_DDS_MASK GENMASK(19, 16)
#define CHCFG_FILL_SDS_MASK GENMASK(15, 12)
#define CHCFG_FILL_TM(a) (((a) & BIT(5)) << 22)
#define CHCFG_FILL_AM(a) (((a) & GENMASK(4, 2)) << 6)
#define CHCFG_FILL_LVL(a) (((a) & BIT(1)) << 5)
#define CHCFG_FILL_HIEN(a) (((a) & BIT(0)) << 5)
#define MID_RID_MASK GENMASK(9, 0)
#define CHCFG_MASK GENMASK(15, 10)
#define CHCFG_DS_INVALID 0xFF
#define DCTRL_LVINT BIT(1)
#define DCTRL_PR BIT(0)
#define DCTRL_DEFAULT (DCTRL_LVINT | DCTRL_PR)
#define HEADER_LV BIT(0)
#define HEADER_LE BIT(1)
#define HEADER_WBD BIT(2)
#define RZ_DMAC_MAX_CHAN_DESCRIPTORS 16
#define RZ_DMAC_MAX_CHANNELS 16
#define DMAC_NR_LMDESC 64
#define RZV2H_MAX_DMAC_INDEX 4
static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val,
unsigned int offset)
{
writel(val, dmac->base + offset);
}
static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val,
unsigned int offset)
{
writel(val, dmac->ext_base + offset);
}
static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset)
{
return readl(dmac->ext_base + offset);
}
static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val,
unsigned int offset, int which)
{
if (which)
writel(val, channel->ch_base + offset);
else
writel(val, channel->ch_cmn_base + offset);
}
static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel,
unsigned int offset, int which)
{
if (which)
return readl(channel->ch_base + offset);
else
return readl(channel->ch_cmn_base + offset);
}
static void rz_lmdesc_setup(struct rz_dmac_chan *channel,
struct rz_lmdesc *lmdesc)
{
u32 nxla;
channel->lmdesc.base = lmdesc;
channel->lmdesc.head = lmdesc;
channel->lmdesc.tail = lmdesc;
nxla = channel->lmdesc.base_dma;
while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) {
lmdesc->header = 0;
nxla += sizeof(*lmdesc);
lmdesc->nxla = nxla;
lmdesc++;
}
lmdesc->header = 0;
lmdesc->nxla = channel->lmdesc.base_dma;
}
static u32 rz_dmac_lmdesc_addr(struct rz_dmac_chan *channel, struct rz_lmdesc *lmdesc)
{
return channel->lmdesc.base_dma +
(sizeof(struct rz_lmdesc) * (lmdesc - channel->lmdesc.base));
}
static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel)
{
struct rz_lmdesc *lmdesc = channel->lmdesc.head;
while (!(lmdesc->header & HEADER_LV)) {
lmdesc->header = 0;
lmdesc++;
if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
lmdesc = channel->lmdesc.base;
}
channel->lmdesc.head = lmdesc;
}
static bool rz_dmac_chan_is_enabled(struct rz_dmac_chan *channel)
{
u32 val = rz_dmac_ch_readl(channel, CHSTAT, 1);
return !!(val & CHSTAT_EN);
}
static bool rz_dmac_chan_is_paused(struct rz_dmac_chan *channel)
{
u32 val = rz_dmac_ch_readl(channel, CHSTAT, 1);
return !!(val & CHSTAT_SUS);
}
static void rz_dmac_enable_hw(struct rz_dmac_chan *channel)
{
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
u32 nxla;
u32 chctrl;
dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
rz_dmac_lmdesc_recycle(channel);
nxla = rz_dmac_lmdesc_addr(channel, channel->lmdesc.head);
chctrl = (channel->chctrl | CHCTRL_SETEN);
rz_dmac_ch_writel(channel, nxla, NXLA, 1);
rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
}
static void rz_dmac_disable_hw(struct rz_dmac_chan *channel)
{
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
}
static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars)
{
u32 dmars_offset = (nr / 2) * 4;
u32 shift = (nr % 2) * 16;
u32 dmars32;
dmars32 = rz_dmac_ext_readl(dmac, dmars_offset);
dmars32 &= ~(0xffff << shift);
dmars32 |= dmars << shift;
rz_dmac_ext_writel(dmac, dmars32, dmars_offset);
}
static void rz_dmac_set_dma_req_no(struct rz_dmac *dmac, unsigned int index,
int req_no)
{
if (dmac->info->icu_register_dma_req)
dmac->info->icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index,
index, req_no);
else
rz_dmac_set_dmars_register(dmac, index, req_no);
}
static int rz_dmac_get_ack_no(const struct rz_dmac_info *info, u16 req_no)
{
if (!info->icu_register_dma_ack)
return -EINVAL;
switch (req_no) {
case 0x000 ... 0x004:
return req_no + 84;
case 0x161 ... 0x198:
return req_no - 0x145;
case 0x199 ... 0x1b4:
return req_no - 0x199;
default:
return -EINVAL;
}
}
static void rz_dmac_set_dma_ack_no(struct rz_dmac *dmac, unsigned int index,
int ack_no)
{
if (ack_no < 0 || !dmac->info->icu_register_dma_ack)
return;
dmac->info->icu_register_dma_ack(dmac->icu.pdev, dmac->icu.dmac_index,
index, ack_no);
}
static void rz_dmac_reset_dma_ack_no(struct rz_dmac *dmac, int ack_no)
{
if (ack_no < 0 || !dmac->info->icu_register_dma_ack)
return;
dmac->info->icu_register_dma_ack(dmac->icu.pdev, dmac->icu.dmac_index,
dmac->info->default_dma_ack_no, ack_no);
}
static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
{
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
struct rz_lmdesc *lmdesc = channel->lmdesc.tail;
struct rz_dmac_desc *d = channel->desc;
u32 chcfg = CHCFG_MEM_COPY;
d->start_lmdesc = lmdesc;
lmdesc->sa = d->src;
lmdesc->da = d->dest;
lmdesc->tb = d->len;
lmdesc->chcfg = chcfg;
lmdesc->chitvl = 0;
lmdesc->chext = 0;
lmdesc->header = HEADER_LV | HEADER_LE;
rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no);
channel->chcfg = chcfg;
channel->chctrl = CHCTRL_STG;
}
static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
{
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
struct rz_dmac_desc *d = channel->desc;
struct scatterlist *sg, *sgl = d->sg;
struct rz_lmdesc *lmdesc;
unsigned int i, sg_len = d->sgcount;
channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS;
if (d->direction == DMA_DEV_TO_MEM) {
channel->chcfg |= CHCFG_SAD;
channel->chcfg &= ~CHCFG_REQD;
} else {
channel->chcfg |= CHCFG_DAD | CHCFG_REQD;
}
lmdesc = channel->lmdesc.tail;
d->start_lmdesc = lmdesc;
for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) {
if (d->direction == DMA_DEV_TO_MEM) {
lmdesc->sa = channel->src_per_address;
lmdesc->da = sg_dma_address(sg);
} else {
lmdesc->sa = sg_dma_address(sg);
lmdesc->da = channel->dst_per_address;
}
lmdesc->tb = sg_dma_len(sg);
lmdesc->chitvl = 0;
lmdesc->chext = 0;
if (i == (sg_len - 1)) {
lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM);
lmdesc->header = HEADER_LV | HEADER_LE;
} else {
lmdesc->chcfg = channel->chcfg;
lmdesc->header = HEADER_LV;
}
if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
lmdesc = channel->lmdesc.base;
}
channel->lmdesc.tail = lmdesc;
rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid);
rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack);
channel->chctrl = 0;
}
static void rz_dmac_prepare_descs_for_cyclic(struct rz_dmac_chan *channel)
{
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
struct rz_dmac_desc *d = channel->desc;
size_t period_len = d->sgcount;
struct rz_lmdesc *lmdesc;
size_t buf_len = d->len;
size_t periods = buf_len / period_len;
lockdep_assert_held(&channel->vc.lock);
channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DMS;
if (d->direction == DMA_DEV_TO_MEM) {
channel->chcfg |= CHCFG_SAD;
channel->chcfg &= ~CHCFG_REQD;
} else {
channel->chcfg |= CHCFG_DAD | CHCFG_REQD;
}
lmdesc = channel->lmdesc.tail;
d->start_lmdesc = lmdesc;
for (size_t i = 0; i < periods; i++) {
if (d->direction == DMA_DEV_TO_MEM) {
lmdesc->sa = d->src;
lmdesc->da = d->dest + (i * period_len);
} else {
lmdesc->sa = d->src + (i * period_len);
lmdesc->da = d->dest;
}
lmdesc->tb = period_len;
lmdesc->chitvl = 0;
lmdesc->chext = 0;
lmdesc->chcfg = channel->chcfg;
lmdesc->header = HEADER_LV | HEADER_WBD;
if (i == periods - 1)
lmdesc->nxla = rz_dmac_lmdesc_addr(channel, d->start_lmdesc);
if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
lmdesc = channel->lmdesc.base;
}
channel->lmdesc.tail = lmdesc;
rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid);
rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack);
}
static void rz_dmac_xfer_desc(struct rz_dmac_chan *chan)
{
struct virt_dma_desc *vd;
vd = vchan_next_desc(&chan->vc);
if (!vd) {
chan->desc = NULL;
return;
}
list_del(&vd->node);
chan->desc = to_rz_dmac_desc(vd);
switch (chan->desc->type) {
case RZ_DMAC_DESC_MEMCPY:
rz_dmac_prepare_desc_for_memcpy(chan);
break;
case RZ_DMAC_DESC_SLAVE_SG:
rz_dmac_prepare_descs_for_slave_sg(chan);
break;
case RZ_DMAC_DESC_CYCLIC:
rz_dmac_prepare_descs_for_cyclic(chan);
break;
}
rz_dmac_enable_hw(chan);
}
static int rz_dmac_alloc_chan_resources(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) {
struct rz_dmac_desc *desc;
desc = kzalloc_obj(*desc);
if (!desc)
break;
list_add_tail(&desc->node, &channel->ld_free);
channel->descs_allocated++;
}
if (!channel->descs_allocated)
return -ENOMEM;
return channel->descs_allocated;
}
static void rz_dmac_free_chan_resources(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
struct rz_dmac_desc *desc, *_desc;
unsigned long flags;
int ret;
PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
if (ret) {
dev_err(dmac->dev, "RPM resume failed for channel %s, ret=%d\n!",
dma_chan_name(chan), ret);
}
spin_lock_irqsave(&channel->vc.lock, flags);
rz_lmdesc_setup(channel, channel->lmdesc.base);
if (!ret)
rz_dmac_disable_hw(channel);
if (channel->mid_rid >= 0) {
clear_bit(channel->mid_rid, dmac->modules);
channel->mid_rid = -EINVAL;
}
channel->status = 0;
rz_dmac_reset_dma_ack_no(dmac, channel->dmac_ack);
channel->dmac_ack = -EINVAL;
spin_unlock_irqrestore(&channel->vc.lock, flags);
vchan_free_chan_resources(&channel->vc);
spin_lock_irqsave(&channel->vc.lock, flags);
list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) {
list_del(&desc->node);
kfree(desc);
channel->descs_allocated--;
}
INIT_LIST_HEAD(&channel->ld_free);
spin_unlock_irqrestore(&channel->vc.lock, flags);
}
static struct dma_async_tx_descriptor *
rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
size_t len, unsigned long flags)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
struct rz_dmac_desc *desc;
dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n",
__func__, channel->index, &src, &dest, len);
scoped_guard(spinlock_irqsave, &channel->vc.lock) {
desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node);
if (!desc)
return NULL;
list_del(&desc->node);
}
desc->type = RZ_DMAC_DESC_MEMCPY;
desc->src = src;
desc->dest = dest;
desc->len = len;
desc->direction = DMA_MEM_TO_MEM;
return vchan_tx_prep(&channel->vc, &desc->vd, flags);
}
static struct dma_async_tx_descriptor *
rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len,
enum dma_transfer_direction direction,
unsigned long flags, void *context)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac_desc *desc;
struct scatterlist *sg;
int dma_length = 0;
int i = 0;
scoped_guard(spinlock_irqsave, &channel->vc.lock) {
desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node);
if (!desc)
return NULL;
list_del(&desc->node);
}
for_each_sg(sgl, sg, sg_len, i)
dma_length += sg_dma_len(sg);
desc->type = RZ_DMAC_DESC_SLAVE_SG;
desc->sg = sgl;
desc->sgcount = sg_len;
desc->len = dma_length;
desc->direction = direction;
if (direction == DMA_DEV_TO_MEM)
desc->src = channel->src_per_address;
else
desc->dest = channel->dst_per_address;
return vchan_tx_prep(&channel->vc, &desc->vd, flags);
}
static struct dma_async_tx_descriptor *
rz_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
size_t buf_len, size_t period_len,
enum dma_transfer_direction direction,
unsigned long flags)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac_desc *desc;
size_t periods;
if (!is_slave_direction(direction))
return NULL;
if (!period_len || !buf_len)
return NULL;
periods = buf_len / period_len;
if (!periods || periods > DMAC_NR_LMDESC)
return NULL;
scoped_guard(spinlock_irqsave, &channel->vc.lock) {
if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC))
return NULL;
desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node);
if (!desc)
return NULL;
list_del(&desc->node);
channel->status |= BIT(RZ_DMAC_CHAN_STATUS_CYCLIC);
}
desc->type = RZ_DMAC_DESC_CYCLIC;
desc->sgcount = period_len;
desc->len = buf_len;
desc->direction = direction;
if (direction == DMA_DEV_TO_MEM) {
desc->src = channel->src_per_address;
desc->dest = buf_addr;
} else {
desc->src = buf_addr;
desc->dest = channel->dst_per_address;
}
return vchan_tx_prep(&channel->vc, &desc->vd, flags);
}
static int rz_dmac_terminate_all(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
unsigned long flags;
LIST_HEAD(head);
int ret;
PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
if (ret) {
dev_err(dmac->dev, "RPM resume failed for channel %s, ret=%d\n!",
dma_chan_name(chan), ret);
}
spin_lock_irqsave(&channel->vc.lock, flags);
if (!ret)
rz_dmac_disable_hw(channel);
rz_lmdesc_setup(channel, channel->lmdesc.base);
if (channel->desc) {
vchan_terminate_vdesc(&channel->desc->vd);
channel->desc = NULL;
}
vchan_get_all_descriptors(&channel->vc, &head);
channel->status = 0;
spin_unlock_irqrestore(&channel->vc.lock, flags);
vchan_dma_desc_free_list(&channel->vc, &head);
return ret;
}
static void rz_dmac_issue_pending(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
unsigned long flags;
int ret;
PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
if (ret)
return;
spin_lock_irqsave(&channel->vc.lock, flags);
if (vchan_issue_pending(&channel->vc) && !channel->desc)
rz_dmac_xfer_desc(channel);
spin_unlock_irqrestore(&channel->vc.lock, flags);
}
static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds)
{
u8 i;
static const enum dma_slave_buswidth ds_lut[] = {
DMA_SLAVE_BUSWIDTH_1_BYTE,
DMA_SLAVE_BUSWIDTH_2_BYTES,
DMA_SLAVE_BUSWIDTH_4_BYTES,
DMA_SLAVE_BUSWIDTH_8_BYTES,
DMA_SLAVE_BUSWIDTH_16_BYTES,
DMA_SLAVE_BUSWIDTH_32_BYTES,
DMA_SLAVE_BUSWIDTH_64_BYTES,
DMA_SLAVE_BUSWIDTH_128_BYTES,
};
for (i = 0; i < ARRAY_SIZE(ds_lut); i++) {
if (ds_lut[i] == ds)
return i;
}
return CHCFG_DS_INVALID;
}
static int rz_dmac_config(struct dma_chan *chan,
struct dma_slave_config *config)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
u32 val;
channel->dst_per_address = config->dst_addr;
channel->chcfg &= ~CHCFG_FILL_DDS_MASK;
if (channel->dst_per_address) {
val = rz_dmac_ds_to_val_mapping(config->dst_addr_width);
if (val == CHCFG_DS_INVALID)
return -EINVAL;
channel->chcfg |= FIELD_PREP(CHCFG_FILL_DDS_MASK, val);
}
channel->src_per_address = config->src_addr;
channel->chcfg &= ~CHCFG_FILL_SDS_MASK;
if (channel->src_per_address) {
val = rz_dmac_ds_to_val_mapping(config->src_addr_width);
if (val == CHCFG_DS_INVALID)
return -EINVAL;
channel->chcfg |= FIELD_PREP(CHCFG_FILL_SDS_MASK, val);
}
return 0;
}
static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(vd->tx.chan);
struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan);
struct rz_dmac_desc *desc = to_rz_dmac_desc(vd);
guard(spinlock_irqsave)(&vc->lock);
list_add_tail(&desc->node, &channel->ld_free);
}
static void rz_dmac_device_synchronize(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
u32 chstat;
int ret;
vchan_synchronize(&channel->vc);
PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
if (ret)
return;
ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN),
100, 100000, false, channel, CHSTAT, 1);
if (ret < 0)
dev_warn(dmac->dev, "DMA Timeout");
rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no);
rz_dmac_reset_dma_ack_no(dmac, channel->dmac_ack);
}
static struct rz_lmdesc *
rz_dmac_get_next_lmdesc(struct rz_lmdesc *base, struct rz_lmdesc *lmdesc)
{
struct rz_lmdesc *next = ++lmdesc;
if (next >= base + DMAC_NR_LMDESC)
next = base;
return next;
}
static u32 rz_dmac_calculate_residue_bytes_in_vd(struct rz_dmac_chan *channel,
struct rz_dmac_desc *desc, u32 crla)
{
struct rz_lmdesc *lmdesc = desc->start_lmdesc;
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
u32 residue = 0, i = 0;
while (lmdesc->nxla != crla) {
lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc);
if (++i >= DMAC_NR_LMDESC)
return 0;
}
if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)) {
u32 start_lmdesc_addr = rz_dmac_lmdesc_addr(channel, desc->start_lmdesc);
while (lmdesc->nxla != start_lmdesc_addr) {
residue += lmdesc->tb;
lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc);
}
} else {
while (lmdesc->chcfg & CHCFG_DEM) {
residue += lmdesc->tb;
lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc);
}
}
dev_dbg(dmac->dev, "%s: VD residue is %u\n", __func__, residue);
return residue;
}
static int rz_dmac_chan_get_residue(struct device *dev, struct rz_dmac_chan *channel,
dma_cookie_t cookie, u32 *residue)
{
struct rz_dmac_desc *desc = NULL;
struct virt_dma_desc *vd;
u32 crla, crtb, i;
int ret;
vd = vchan_find_desc(&channel->vc, cookie);
if (vd) {
desc = to_rz_dmac_desc(vd);
*residue = desc->len;
return 0;
} else if (channel->desc && channel->desc->vd.tx.cookie == cookie) {
desc = channel->desc;
}
if (!desc) {
*residue = 0;
return 0;
}
PM_RUNTIME_ACQUIRE_IF_ENABLED(dev, pm);
ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
if (ret)
return ret;
for (i = 0; i < 3; i++) {
crla = rz_dmac_ch_readl(channel, CRLA, 1);
crtb = rz_dmac_ch_readl(channel, CRTB, 1);
if (crla == rz_dmac_ch_readl(channel, CRLA, 1))
break;
}
WARN_ONCE(i >= 3, "residue might not be continuous!");
*residue = crtb + rz_dmac_calculate_residue_bytes_in_vd(channel, desc, crla);
return 0;
}
static enum dma_status rz_dmac_tx_status(struct dma_chan *chan,
dma_cookie_t cookie,
struct dma_tx_state *txstate)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
enum dma_status status;
u32 residue;
scoped_guard(spinlock_irqsave, &channel->vc.lock) {
int ret;
status = dma_cookie_status(chan, cookie, txstate);
if (status == DMA_COMPLETE || !txstate)
return status;
ret = rz_dmac_chan_get_residue(dmac->dev, channel, cookie, &residue);
if (ret)
return DMA_ERROR;
if (status == DMA_IN_PROGRESS && rz_dmac_chan_is_paused(channel))
status = DMA_PAUSED;
}
dma_set_residue(txstate, residue);
return status;
}
static int rz_dmac_device_pause_set(struct rz_dmac_chan *channel,
unsigned long set_bitmask)
{
int ret = 0;
u32 val;
lockdep_assert_held(&channel->vc.lock);
if (!rz_dmac_chan_is_enabled(channel))
return 0;
if (rz_dmac_chan_is_paused(channel))
goto set_bit;
rz_dmac_ch_writel(channel, CHCTRL_SETSUS, CHCTRL, 1);
ret = read_poll_timeout_atomic(rz_dmac_ch_readl, val,
(val & CHSTAT_SUS), 1, 1024, false,
channel, CHSTAT, 1);
set_bit:
channel->status |= set_bitmask;
return ret;
}
static int rz_dmac_device_pause(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
int ret;
PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
if (ret)
return ret;
guard(spinlock_irqsave)(&channel->vc.lock);
return rz_dmac_device_pause_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED));
}
static int rz_dmac_device_pause_internal(struct rz_dmac_chan *channel)
{
lockdep_assert_held(&channel->vc.lock);
if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED) ||
!rz_dmac_chan_is_enabled(channel))
return 0;
return rz_dmac_device_pause_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL));
}
static int rz_dmac_device_resume_set(struct rz_dmac_chan *channel,
unsigned long clear_bitmask)
{
u32 val;
int ret;
lockdep_assert_held(&channel->vc.lock);
rz_dmac_ch_writel(channel, CHCTRL_CLRSUS | CHCTRL_SETEN, CHCTRL, 1);
ret = read_poll_timeout_atomic(rz_dmac_ch_readl, val,
((val & (CHSTAT_SUS | CHSTAT_EN)) == CHSTAT_EN),
1, 1024, false, channel, CHSTAT, 1);
channel->status &= ~clear_bitmask;
return ret;
}
static int rz_dmac_device_resume(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
int ret;
PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
if (ret)
return ret;
guard(spinlock_irqsave)(&channel->vc.lock);
if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED)))
return 0;
return rz_dmac_device_resume_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED));
}
static int rz_dmac_device_resume_internal(struct rz_dmac_chan *channel)
{
lockdep_assert_held(&channel->vc.lock);
if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL)))
return 0;
return rz_dmac_device_resume_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL));
}
static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel)
{
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
u32 chstat;
chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
if (chstat & CHSTAT_ER) {
dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n",
channel->index, chstat);
scoped_guard(spinlock_irqsave, &channel->vc.lock)
rz_dmac_disable_hw(channel);
return;
}
rz_dmac_ch_writel(channel, CHCTRL_CLREND, CHCTRL, 1);
}
static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id)
{
struct rz_dmac_chan *channel = dev_id;
if (channel) {
rz_dmac_irq_handle_channel(channel);
return IRQ_WAKE_THREAD;
}
return IRQ_HANDLED;
}
static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id)
{
struct rz_dmac_chan *channel = dev_id;
struct rz_dmac_desc *desc;
guard(spinlock_irqsave)(&channel->vc.lock);
desc = channel->desc;
if (!desc)
return IRQ_HANDLED;
if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)) {
vchan_cyclic_callback(&desc->vd);
} else {
vchan_cookie_complete(&desc->vd);
channel->desc = NULL;
rz_dmac_xfer_desc(channel);
}
return IRQ_HANDLED;
}
static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
struct of_phandle_args *dma_spec = arg;
u32 ch_cfg;
channel->mid_rid = dma_spec->args[0] & MID_RID_MASK;
ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10;
channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) |
CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg);
channel->dmac_ack = rz_dmac_get_ack_no(dmac->info, channel->mid_rid);
return !test_and_set_bit(channel->mid_rid, dmac->modules);
}
static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec,
struct of_dma *ofdma)
{
dma_cap_mask_t mask;
if (dma_spec->args_count != 1)
return NULL;
dma_cap_zero(mask);
dma_cap_set(DMA_SLAVE, mask);
return __dma_request_channel(&mask, rz_dmac_chan_filter, dma_spec,
ofdma->of_node);
}
static int rz_dmac_chan_probe(struct rz_dmac *dmac,
struct rz_dmac_chan *channel,
u8 index)
{
struct platform_device *pdev = to_platform_device(dmac->dev);
struct rz_lmdesc *lmdesc;
char pdev_irqname[6];
char *irqname;
int irq, ret;
channel->index = index;
channel->mid_rid = -EINVAL;
channel->dmac_ack = -EINVAL;
if (index < 8) {
channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET +
EACH_CHANNEL_OFFSET * index;
channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE;
} else {
channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET +
EACH_CHANNEL_OFFSET * (index - 8);
channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE;
}
lmdesc = dmam_alloc_coherent(&pdev->dev,
sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
&channel->lmdesc.base_dma, GFP_KERNEL);
if (!lmdesc) {
dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n");
return -ENOMEM;
}
rz_lmdesc_setup(channel, lmdesc);
channel->vc.desc_free = rz_dmac_virt_desc_free;
vchan_init(&channel->vc, &dmac->engine);
INIT_LIST_HEAD(&channel->ld_free);
rz_dmac_disable_hw(channel);
scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index);
irq = platform_get_irq_byname(pdev, pdev_irqname);
if (irq < 0)
return irq;
irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
dev_name(dmac->dev), index);
if (!irqname)
return -ENOMEM;
ret = devm_request_threaded_irq(dmac->dev, irq, rz_dmac_irq_handler,
rz_dmac_irq_handler_thread, 0,
irqname, channel);
if (ret)
dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", irq, ret);
return ret;
}
static void rz_dmac_put_device(void *_dev)
{
struct device *dev = _dev;
put_device(dev);
}
static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac)
{
struct device_node *np = dev->of_node;
struct of_phandle_args args;
uint32_t dmac_index;
int ret;
if (!dmac->info->icu_register_dma_req)
return 0;
ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args);
if (ret)
return ret;
dmac->icu.pdev = of_find_device_by_node(args.np);
of_node_put(args.np);
if (!dmac->icu.pdev) {
dev_err(dev, "ICU device not found.\n");
return -ENODEV;
}
ret = devm_add_action_or_reset(dev, rz_dmac_put_device, &dmac->icu.pdev->dev);
if (ret)
return ret;
dmac_index = args.args[0];
if (dmac_index > RZV2H_MAX_DMAC_INDEX) {
dev_err(dev, "DMAC index %u invalid.\n", dmac_index);
return -EINVAL;
}
dmac->icu.dmac_index = dmac_index;
return 0;
}
static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac)
{
struct device_node *np = dev->of_node;
int ret;
ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels);
if (ret < 0) {
dev_err(dev, "unable to read dma-channels property\n");
return ret;
}
if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) {
dev_err(dev, "invalid number of channels %u\n", dmac->n_channels);
return -EINVAL;
}
return rz_dmac_parse_of_icu(dev, dmac);
}
static int rz_dmac_probe(struct platform_device *pdev)
{
const char *irqname = "error";
struct dma_device *engine;
struct rz_dmac *dmac;
int ret;
int irq;
u8 i;
dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
if (!dmac)
return -ENOMEM;
dmac->info = device_get_match_data(&pdev->dev);
dmac->dev = &pdev->dev;
platform_set_drvdata(pdev, dmac);
ret = rz_dmac_parse_of(&pdev->dev, dmac);
if (ret < 0)
return ret;
dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels,
sizeof(*dmac->channels), GFP_KERNEL);
if (!dmac->channels)
return -ENOMEM;
dmac->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(dmac->base))
return PTR_ERR(dmac->base);
if (!dmac->info->icu_register_dma_req) {
dmac->ext_base = devm_platform_ioremap_resource(pdev, 1);
if (IS_ERR(dmac->ext_base))
return PTR_ERR(dmac->ext_base);
}
INIT_LIST_HEAD(&dmac->engine.channels);
dmac->rstc = devm_reset_control_array_get_optional_exclusive(&pdev->dev);
if (IS_ERR(dmac->rstc))
return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc),
"failed to get resets\n");
pm_runtime_irq_safe(&pdev->dev);
pm_runtime_enable(&pdev->dev);
ret = pm_runtime_resume_and_get(&pdev->dev);
if (ret < 0) {
dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n");
goto err_pm_disable;
}
ret = reset_control_deassert(dmac->rstc);
if (ret)
goto err_pm_runtime_put;
for (i = 0; i < dmac->n_channels; i++) {
ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i);
if (ret < 0)
goto err;
}
irq = platform_get_irq_byname_optional(pdev, irqname);
if (irq > 0) {
ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
irqname, NULL);
if (ret) {
dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
irq, ret);
goto err;
}
}
ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate,
NULL);
if (ret < 0)
goto err;
engine = &dmac->engine;
dma_cap_set(DMA_SLAVE, engine->cap_mask);
dma_cap_set(DMA_MEMCPY, engine->cap_mask);
dma_cap_set(DMA_CYCLIC, engine->cap_mask);
engine->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL);
rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL);
engine->dev = &pdev->dev;
engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources;
engine->device_free_chan_resources = rz_dmac_free_chan_resources;
engine->device_tx_status = rz_dmac_tx_status;
engine->device_prep_slave_sg = rz_dmac_prep_slave_sg;
engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy;
engine->device_prep_dma_cyclic = rz_dmac_prep_dma_cyclic;
engine->device_config = rz_dmac_config;
engine->device_terminate_all = rz_dmac_terminate_all;
engine->device_issue_pending = rz_dmac_issue_pending;
engine->device_synchronize = rz_dmac_device_synchronize;
engine->device_pause = rz_dmac_device_pause;
engine->device_resume = rz_dmac_device_resume;
engine->copy_align = DMAENGINE_ALIGN_1_BYTE;
dma_set_max_seg_size(engine->dev, U32_MAX);
ret = dma_async_device_register(engine);
if (ret < 0) {
dev_err(&pdev->dev, "unable to register\n");
goto dma_register_err;
}
return 0;
dma_register_err:
of_dma_controller_free(pdev->dev.of_node);
err:
reset_control_assert(dmac->rstc);
err_pm_runtime_put:
pm_runtime_put(&pdev->dev);
err_pm_disable:
pm_runtime_disable(&pdev->dev);
return ret;
}
static void rz_dmac_remove(struct platform_device *pdev)
{
struct rz_dmac *dmac = platform_get_drvdata(pdev);
dma_async_device_unregister(&dmac->engine);
of_dma_controller_free(pdev->dev.of_node);
reset_control_assert(dmac->rstc);
pm_runtime_put(&pdev->dev);
pm_runtime_disable(&pdev->dev);
}
static void rz_dmac_suspend_recover(struct rz_dmac *dmac)
{
int ret;
PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm);
ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
if (ret)
return;
for (unsigned int i = 0; i < dmac->n_channels; i++) {
struct rz_dmac_chan *channel = &dmac->channels[i];
guard(spinlock_irqsave)(&channel->vc.lock);
if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)))
continue;
rz_dmac_device_resume_internal(channel);
}
}
static int rz_dmac_suspend(struct device *dev)
{
struct rz_dmac *dmac = dev_get_drvdata(dev);
int ret = 0;
for (unsigned int i = 0; i < dmac->n_channels; i++) {
struct rz_dmac_chan *channel = &dmac->channels[i];
guard(spinlock_irqsave)(&channel->vc.lock);
if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)))
continue;
ret = rz_dmac_device_pause_internal(channel);
if (ret) {
dev_err(dev, "Failed to suspend channel %s\n",
dma_chan_name(&channel->vc.chan));
break;
}
channel->pm_state.nxla = rz_dmac_ch_readl(channel, NXLA, 1);
}
if (ret)
goto suspend_recover;
ret = reset_control_assert(dmac->rstc);
if (ret)
goto suspend_recover;
ret = pm_runtime_put_sync(dev);
if (ret < 0)
goto reset_deassert;
return 0;
reset_deassert:
reset_control_deassert(dmac->rstc);
suspend_recover:
rz_dmac_suspend_recover(dmac);
return ret;
}
static int rz_dmac_resume(struct device *dev)
{
struct rz_dmac *dmac = dev_get_drvdata(dev);
int errors = 0, ret;
ret = pm_runtime_resume_and_get(dev);
if (ret)
return ret;
ret = reset_control_deassert(dmac->rstc);
if (ret) {
return ret;
}
rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL);
rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL);
for (unsigned int i = 0; i < dmac->n_channels; i++) {
struct rz_dmac_chan *channel = &dmac->channels[i];
guard(spinlock_irqsave)(&channel->vc.lock);
rz_dmac_disable_hw(&dmac->channels[i]);
if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)))
continue;
rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid);
rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack);
rz_dmac_ch_writel(channel, channel->pm_state.nxla, NXLA, 1);
rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
rz_dmac_ch_writel(channel, channel->chctrl, CHCTRL, 1);
ret = rz_dmac_device_resume_internal(channel);
if (ret) {
errors = ret;
dev_err(dev, "Failed to resume channel %s, ret=%d\n",
dma_chan_name(&channel->vc.chan), ret);
}
}
return errors ? : 0;
}
static DEFINE_SIMPLE_DEV_PM_OPS(rz_dmac_pm_ops, rz_dmac_suspend, rz_dmac_resume);
static const struct rz_dmac_info rz_dmac_v2h_info = {
.icu_register_dma_req = rzv2h_icu_register_dma_req,
.icu_register_dma_ack = rzv2h_icu_register_dma_ack,
.default_dma_ack_no = RZV2H_ICU_DMAC_ACK_NO_DEFAULT,
.default_dma_req_no = RZV2H_ICU_DMAC_REQ_NO_DEFAULT,
};
static const struct rz_dmac_info rz_dmac_t2h_info = {
.icu_register_dma_req = rzt2h_icu_register_dma_req,
.default_dma_req_no = RZT2H_ICU_DMAC_REQ_NO_DEFAULT,
};
static const struct rz_dmac_info rz_dmac_generic_info = {
.default_dma_req_no = 0,
};
static const struct of_device_id of_rz_dmac_match[] = {
{ .compatible = "renesas,r9a09g057-dmac", .data = &rz_dmac_v2h_info },
{ .compatible = "renesas,r9a09g077-dmac", .data = &rz_dmac_t2h_info },
{ .compatible = "renesas,rz-dmac", .data = &rz_dmac_generic_info },
{ }
};
MODULE_DEVICE_TABLE(of, of_rz_dmac_match);
static struct platform_driver rz_dmac_driver = {
.driver = {
.name = "rz-dmac",
.of_match_table = of_rz_dmac_match,
.pm = pm_ptr(&rz_dmac_pm_ops),
},
.probe = rz_dmac_probe,
.remove = rz_dmac_remove,
};
module_platform_driver(rz_dmac_driver);
MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver");
MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>");
MODULE_LICENSE("GPL v2");
