forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1090 lines
28 KiB
1090 lines
28 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* |
|
* IMG Multi-threaded DMA Controller (MDC) |
|
* |
|
* Copyright (C) 2009,2012,2013 Imagination Technologies Ltd. |
|
* Copyright (C) 2014 Google, Inc. |
|
*/ |
|
|
|
#include <linux/clk.h> |
|
#include <linux/dma-mapping.h> |
|
#include <linux/dmaengine.h> |
|
#include <linux/dmapool.h> |
|
#include <linux/interrupt.h> |
|
#include <linux/io.h> |
|
#include <linux/irq.h> |
|
#include <linux/kernel.h> |
|
#include <linux/mfd/syscon.h> |
|
#include <linux/module.h> |
|
#include <linux/of.h> |
|
#include <linux/of_device.h> |
|
#include <linux/of_dma.h> |
|
#include <linux/platform_device.h> |
|
#include <linux/pm_runtime.h> |
|
#include <linux/regmap.h> |
|
#include <linux/slab.h> |
|
#include <linux/spinlock.h> |
|
|
|
#include "dmaengine.h" |
|
#include "virt-dma.h" |
|
|
|
#define MDC_MAX_DMA_CHANNELS 32 |
|
|
|
#define MDC_GENERAL_CONFIG 0x000 |
|
#define MDC_GENERAL_CONFIG_LIST_IEN BIT(31) |
|
#define MDC_GENERAL_CONFIG_IEN BIT(29) |
|
#define MDC_GENERAL_CONFIG_LEVEL_INT BIT(28) |
|
#define MDC_GENERAL_CONFIG_INC_W BIT(12) |
|
#define MDC_GENERAL_CONFIG_INC_R BIT(8) |
|
#define MDC_GENERAL_CONFIG_PHYSICAL_W BIT(7) |
|
#define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT 4 |
|
#define MDC_GENERAL_CONFIG_WIDTH_W_MASK 0x7 |
|
#define MDC_GENERAL_CONFIG_PHYSICAL_R BIT(3) |
|
#define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT 0 |
|
#define MDC_GENERAL_CONFIG_WIDTH_R_MASK 0x7 |
|
|
|
#define MDC_READ_PORT_CONFIG 0x004 |
|
#define MDC_READ_PORT_CONFIG_STHREAD_SHIFT 28 |
|
#define MDC_READ_PORT_CONFIG_STHREAD_MASK 0xf |
|
#define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT 24 |
|
#define MDC_READ_PORT_CONFIG_RTHREAD_MASK 0xf |
|
#define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT 16 |
|
#define MDC_READ_PORT_CONFIG_WTHREAD_MASK 0xf |
|
#define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT 4 |
|
#define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK 0xff |
|
#define MDC_READ_PORT_CONFIG_DREQ_ENABLE BIT(1) |
|
|
|
#define MDC_READ_ADDRESS 0x008 |
|
|
|
#define MDC_WRITE_ADDRESS 0x00c |
|
|
|
#define MDC_TRANSFER_SIZE 0x010 |
|
#define MDC_TRANSFER_SIZE_MASK 0xffffff |
|
|
|
#define MDC_LIST_NODE_ADDRESS 0x014 |
|
|
|
#define MDC_CMDS_PROCESSED 0x018 |
|
#define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16 |
|
#define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK 0x3f |
|
#define MDC_CMDS_PROCESSED_INT_ACTIVE BIT(8) |
|
#define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT 0 |
|
#define MDC_CMDS_PROCESSED_CMDS_DONE_MASK 0x3f |
|
|
|
#define MDC_CONTROL_AND_STATUS 0x01c |
|
#define MDC_CONTROL_AND_STATUS_CANCEL BIT(20) |
|
#define MDC_CONTROL_AND_STATUS_LIST_EN BIT(4) |
|
#define MDC_CONTROL_AND_STATUS_EN BIT(0) |
|
|
|
#define MDC_ACTIVE_TRANSFER_SIZE 0x030 |
|
|
|
#define MDC_GLOBAL_CONFIG_A 0x900 |
|
#define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT 16 |
|
#define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK 0xff |
|
#define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT 8 |
|
#define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK 0xff |
|
#define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT 0 |
|
#define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK 0xff |
|
|
|
struct mdc_hw_list_desc { |
|
u32 gen_conf; |
|
u32 readport_conf; |
|
u32 read_addr; |
|
u32 write_addr; |
|
u32 xfer_size; |
|
u32 node_addr; |
|
u32 cmds_done; |
|
u32 ctrl_status; |
|
/* |
|
* Not part of the list descriptor, but instead used by the CPU to |
|
* traverse the list. |
|
*/ |
|
struct mdc_hw_list_desc *next_desc; |
|
}; |
|
|
|
struct mdc_tx_desc { |
|
struct mdc_chan *chan; |
|
struct virt_dma_desc vd; |
|
dma_addr_t list_phys; |
|
struct mdc_hw_list_desc *list; |
|
bool cyclic; |
|
bool cmd_loaded; |
|
unsigned int list_len; |
|
unsigned int list_period_len; |
|
size_t list_xfer_size; |
|
unsigned int list_cmds_done; |
|
}; |
|
|
|
struct mdc_chan { |
|
struct mdc_dma *mdma; |
|
struct virt_dma_chan vc; |
|
struct dma_slave_config config; |
|
struct mdc_tx_desc *desc; |
|
int irq; |
|
unsigned int periph; |
|
unsigned int thread; |
|
unsigned int chan_nr; |
|
}; |
|
|
|
struct mdc_dma_soc_data { |
|
void (*enable_chan)(struct mdc_chan *mchan); |
|
void (*disable_chan)(struct mdc_chan *mchan); |
|
}; |
|
|
|
struct mdc_dma { |
|
struct dma_device dma_dev; |
|
void __iomem *regs; |
|
struct clk *clk; |
|
struct dma_pool *desc_pool; |
|
struct regmap *periph_regs; |
|
spinlock_t lock; |
|
unsigned int nr_threads; |
|
unsigned int nr_channels; |
|
unsigned int bus_width; |
|
unsigned int max_burst_mult; |
|
unsigned int max_xfer_size; |
|
const struct mdc_dma_soc_data *soc; |
|
struct mdc_chan channels[MDC_MAX_DMA_CHANNELS]; |
|
}; |
|
|
|
static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg) |
|
{ |
|
return readl(mdma->regs + reg); |
|
} |
|
|
|
static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg) |
|
{ |
|
writel(val, mdma->regs + reg); |
|
} |
|
|
|
static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg) |
|
{ |
|
return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg); |
|
} |
|
|
|
static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg) |
|
{ |
|
mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg); |
|
} |
|
|
|
static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c) |
|
{ |
|
return container_of(to_virt_chan(c), struct mdc_chan, vc); |
|
} |
|
|
|
static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t) |
|
{ |
|
struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx); |
|
|
|
return container_of(vdesc, struct mdc_tx_desc, vd); |
|
} |
|
|
|
static inline struct device *mdma2dev(struct mdc_dma *mdma) |
|
{ |
|
return mdma->dma_dev.dev; |
|
} |
|
|
|
static inline unsigned int to_mdc_width(unsigned int bytes) |
|
{ |
|
return ffs(bytes) - 1; |
|
} |
|
|
|
static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc, |
|
unsigned int bytes) |
|
{ |
|
ldesc->gen_conf |= to_mdc_width(bytes) << |
|
MDC_GENERAL_CONFIG_WIDTH_R_SHIFT; |
|
} |
|
|
|
static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc, |
|
unsigned int bytes) |
|
{ |
|
ldesc->gen_conf |= to_mdc_width(bytes) << |
|
MDC_GENERAL_CONFIG_WIDTH_W_SHIFT; |
|
} |
|
|
|
static void mdc_list_desc_config(struct mdc_chan *mchan, |
|
struct mdc_hw_list_desc *ldesc, |
|
enum dma_transfer_direction dir, |
|
dma_addr_t src, dma_addr_t dst, size_t len) |
|
{ |
|
struct mdc_dma *mdma = mchan->mdma; |
|
unsigned int max_burst, burst_size; |
|
|
|
ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN | |
|
MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W | |
|
MDC_GENERAL_CONFIG_PHYSICAL_R; |
|
ldesc->readport_conf = |
|
(mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) | |
|
(mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) | |
|
(mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT); |
|
ldesc->read_addr = src; |
|
ldesc->write_addr = dst; |
|
ldesc->xfer_size = len - 1; |
|
ldesc->node_addr = 0; |
|
ldesc->cmds_done = 0; |
|
ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN | |
|
MDC_CONTROL_AND_STATUS_EN; |
|
ldesc->next_desc = NULL; |
|
|
|
if (IS_ALIGNED(dst, mdma->bus_width) && |
|
IS_ALIGNED(src, mdma->bus_width)) |
|
max_burst = mdma->bus_width * mdma->max_burst_mult; |
|
else |
|
max_burst = mdma->bus_width * (mdma->max_burst_mult - 1); |
|
|
|
if (dir == DMA_MEM_TO_DEV) { |
|
ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R; |
|
ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE; |
|
mdc_set_read_width(ldesc, mdma->bus_width); |
|
mdc_set_write_width(ldesc, mchan->config.dst_addr_width); |
|
burst_size = min(max_burst, mchan->config.dst_maxburst * |
|
mchan->config.dst_addr_width); |
|
} else if (dir == DMA_DEV_TO_MEM) { |
|
ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W; |
|
ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE; |
|
mdc_set_read_width(ldesc, mchan->config.src_addr_width); |
|
mdc_set_write_width(ldesc, mdma->bus_width); |
|
burst_size = min(max_burst, mchan->config.src_maxburst * |
|
mchan->config.src_addr_width); |
|
} else { |
|
ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R | |
|
MDC_GENERAL_CONFIG_INC_W; |
|
mdc_set_read_width(ldesc, mdma->bus_width); |
|
mdc_set_write_width(ldesc, mdma->bus_width); |
|
burst_size = max_burst; |
|
} |
|
ldesc->readport_conf |= (burst_size - 1) << |
|
MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT; |
|
} |
|
|
|
static void mdc_list_desc_free(struct mdc_tx_desc *mdesc) |
|
{ |
|
struct mdc_dma *mdma = mdesc->chan->mdma; |
|
struct mdc_hw_list_desc *curr, *next; |
|
dma_addr_t curr_phys, next_phys; |
|
|
|
curr = mdesc->list; |
|
curr_phys = mdesc->list_phys; |
|
while (curr) { |
|
next = curr->next_desc; |
|
next_phys = curr->node_addr; |
|
dma_pool_free(mdma->desc_pool, curr, curr_phys); |
|
curr = next; |
|
curr_phys = next_phys; |
|
} |
|
} |
|
|
|
static void mdc_desc_free(struct virt_dma_desc *vd) |
|
{ |
|
struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx); |
|
|
|
mdc_list_desc_free(mdesc); |
|
kfree(mdesc); |
|
} |
|
|
|
static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy( |
|
struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len, |
|
unsigned long flags) |
|
{ |
|
struct mdc_chan *mchan = to_mdc_chan(chan); |
|
struct mdc_dma *mdma = mchan->mdma; |
|
struct mdc_tx_desc *mdesc; |
|
struct mdc_hw_list_desc *curr, *prev = NULL; |
|
dma_addr_t curr_phys; |
|
|
|
if (!len) |
|
return NULL; |
|
|
|
mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT); |
|
if (!mdesc) |
|
return NULL; |
|
mdesc->chan = mchan; |
|
mdesc->list_xfer_size = len; |
|
|
|
while (len > 0) { |
|
size_t xfer_size; |
|
|
|
curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys); |
|
if (!curr) |
|
goto free_desc; |
|
|
|
if (prev) { |
|
prev->node_addr = curr_phys; |
|
prev->next_desc = curr; |
|
} else { |
|
mdesc->list_phys = curr_phys; |
|
mdesc->list = curr; |
|
} |
|
|
|
xfer_size = min_t(size_t, mdma->max_xfer_size, len); |
|
|
|
mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest, |
|
xfer_size); |
|
|
|
prev = curr; |
|
|
|
mdesc->list_len++; |
|
src += xfer_size; |
|
dest += xfer_size; |
|
len -= xfer_size; |
|
} |
|
|
|
return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags); |
|
|
|
free_desc: |
|
mdc_desc_free(&mdesc->vd); |
|
|
|
return NULL; |
|
} |
|
|
|
static int mdc_check_slave_width(struct mdc_chan *mchan, |
|
enum dma_transfer_direction dir) |
|
{ |
|
enum dma_slave_buswidth width; |
|
|
|
if (dir == DMA_MEM_TO_DEV) |
|
width = mchan->config.dst_addr_width; |
|
else |
|
width = mchan->config.src_addr_width; |
|
|
|
switch (width) { |
|
case DMA_SLAVE_BUSWIDTH_1_BYTE: |
|
case DMA_SLAVE_BUSWIDTH_2_BYTES: |
|
case DMA_SLAVE_BUSWIDTH_4_BYTES: |
|
case DMA_SLAVE_BUSWIDTH_8_BYTES: |
|
break; |
|
default: |
|
return -EINVAL; |
|
} |
|
|
|
if (width > mchan->mdma->bus_width) |
|
return -EINVAL; |
|
|
|
return 0; |
|
} |
|
|
|
static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic( |
|
struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, |
|
size_t period_len, enum dma_transfer_direction dir, |
|
unsigned long flags) |
|
{ |
|
struct mdc_chan *mchan = to_mdc_chan(chan); |
|
struct mdc_dma *mdma = mchan->mdma; |
|
struct mdc_tx_desc *mdesc; |
|
struct mdc_hw_list_desc *curr, *prev = NULL; |
|
dma_addr_t curr_phys; |
|
|
|
if (!buf_len && !period_len) |
|
return NULL; |
|
|
|
if (!is_slave_direction(dir)) |
|
return NULL; |
|
|
|
if (mdc_check_slave_width(mchan, dir) < 0) |
|
return NULL; |
|
|
|
mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT); |
|
if (!mdesc) |
|
return NULL; |
|
mdesc->chan = mchan; |
|
mdesc->cyclic = true; |
|
mdesc->list_xfer_size = buf_len; |
|
mdesc->list_period_len = DIV_ROUND_UP(period_len, |
|
mdma->max_xfer_size); |
|
|
|
while (buf_len > 0) { |
|
size_t remainder = min(period_len, buf_len); |
|
|
|
while (remainder > 0) { |
|
size_t xfer_size; |
|
|
|
curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, |
|
&curr_phys); |
|
if (!curr) |
|
goto free_desc; |
|
|
|
if (!prev) { |
|
mdesc->list_phys = curr_phys; |
|
mdesc->list = curr; |
|
} else { |
|
prev->node_addr = curr_phys; |
|
prev->next_desc = curr; |
|
} |
|
|
|
xfer_size = min_t(size_t, mdma->max_xfer_size, |
|
remainder); |
|
|
|
if (dir == DMA_MEM_TO_DEV) { |
|
mdc_list_desc_config(mchan, curr, dir, |
|
buf_addr, |
|
mchan->config.dst_addr, |
|
xfer_size); |
|
} else { |
|
mdc_list_desc_config(mchan, curr, dir, |
|
mchan->config.src_addr, |
|
buf_addr, |
|
xfer_size); |
|
} |
|
|
|
prev = curr; |
|
|
|
mdesc->list_len++; |
|
buf_addr += xfer_size; |
|
buf_len -= xfer_size; |
|
remainder -= xfer_size; |
|
} |
|
} |
|
prev->node_addr = mdesc->list_phys; |
|
|
|
return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags); |
|
|
|
free_desc: |
|
mdc_desc_free(&mdesc->vd); |
|
|
|
return NULL; |
|
} |
|
|
|
static struct dma_async_tx_descriptor *mdc_prep_slave_sg( |
|
struct dma_chan *chan, struct scatterlist *sgl, |
|
unsigned int sg_len, enum dma_transfer_direction dir, |
|
unsigned long flags, void *context) |
|
{ |
|
struct mdc_chan *mchan = to_mdc_chan(chan); |
|
struct mdc_dma *mdma = mchan->mdma; |
|
struct mdc_tx_desc *mdesc; |
|
struct scatterlist *sg; |
|
struct mdc_hw_list_desc *curr, *prev = NULL; |
|
dma_addr_t curr_phys; |
|
unsigned int i; |
|
|
|
if (!sgl) |
|
return NULL; |
|
|
|
if (!is_slave_direction(dir)) |
|
return NULL; |
|
|
|
if (mdc_check_slave_width(mchan, dir) < 0) |
|
return NULL; |
|
|
|
mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT); |
|
if (!mdesc) |
|
return NULL; |
|
mdesc->chan = mchan; |
|
|
|
for_each_sg(sgl, sg, sg_len, i) { |
|
dma_addr_t buf = sg_dma_address(sg); |
|
size_t buf_len = sg_dma_len(sg); |
|
|
|
while (buf_len > 0) { |
|
size_t xfer_size; |
|
|
|
curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, |
|
&curr_phys); |
|
if (!curr) |
|
goto free_desc; |
|
|
|
if (!prev) { |
|
mdesc->list_phys = curr_phys; |
|
mdesc->list = curr; |
|
} else { |
|
prev->node_addr = curr_phys; |
|
prev->next_desc = curr; |
|
} |
|
|
|
xfer_size = min_t(size_t, mdma->max_xfer_size, |
|
buf_len); |
|
|
|
if (dir == DMA_MEM_TO_DEV) { |
|
mdc_list_desc_config(mchan, curr, dir, buf, |
|
mchan->config.dst_addr, |
|
xfer_size); |
|
} else { |
|
mdc_list_desc_config(mchan, curr, dir, |
|
mchan->config.src_addr, |
|
buf, xfer_size); |
|
} |
|
|
|
prev = curr; |
|
|
|
mdesc->list_len++; |
|
mdesc->list_xfer_size += xfer_size; |
|
buf += xfer_size; |
|
buf_len -= xfer_size; |
|
} |
|
} |
|
|
|
return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags); |
|
|
|
free_desc: |
|
mdc_desc_free(&mdesc->vd); |
|
|
|
return NULL; |
|
} |
|
|
|
static void mdc_issue_desc(struct mdc_chan *mchan) |
|
{ |
|
struct mdc_dma *mdma = mchan->mdma; |
|
struct virt_dma_desc *vd; |
|
struct mdc_tx_desc *mdesc; |
|
u32 val; |
|
|
|
vd = vchan_next_desc(&mchan->vc); |
|
if (!vd) |
|
return; |
|
|
|
list_del(&vd->node); |
|
|
|
mdesc = to_mdc_desc(&vd->tx); |
|
mchan->desc = mdesc; |
|
|
|
dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n", |
|
mchan->chan_nr); |
|
|
|
mdma->soc->enable_chan(mchan); |
|
|
|
val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG); |
|
val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN | |
|
MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W | |
|
MDC_GENERAL_CONFIG_PHYSICAL_R; |
|
mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG); |
|
val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) | |
|
(mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) | |
|
(mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT); |
|
mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG); |
|
mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS); |
|
val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS); |
|
val |= MDC_CONTROL_AND_STATUS_LIST_EN; |
|
mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS); |
|
} |
|
|
|
static void mdc_issue_pending(struct dma_chan *chan) |
|
{ |
|
struct mdc_chan *mchan = to_mdc_chan(chan); |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&mchan->vc.lock, flags); |
|
if (vchan_issue_pending(&mchan->vc) && !mchan->desc) |
|
mdc_issue_desc(mchan); |
|
spin_unlock_irqrestore(&mchan->vc.lock, flags); |
|
} |
|
|
|
static enum dma_status mdc_tx_status(struct dma_chan *chan, |
|
dma_cookie_t cookie, struct dma_tx_state *txstate) |
|
{ |
|
struct mdc_chan *mchan = to_mdc_chan(chan); |
|
struct mdc_tx_desc *mdesc; |
|
struct virt_dma_desc *vd; |
|
unsigned long flags; |
|
size_t bytes = 0; |
|
int ret; |
|
|
|
ret = dma_cookie_status(chan, cookie, txstate); |
|
if (ret == DMA_COMPLETE) |
|
return ret; |
|
|
|
if (!txstate) |
|
return ret; |
|
|
|
spin_lock_irqsave(&mchan->vc.lock, flags); |
|
vd = vchan_find_desc(&mchan->vc, cookie); |
|
if (vd) { |
|
mdesc = to_mdc_desc(&vd->tx); |
|
bytes = mdesc->list_xfer_size; |
|
} else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) { |
|
struct mdc_hw_list_desc *ldesc; |
|
u32 val1, val2, done, processed, residue; |
|
int i, cmds; |
|
|
|
mdesc = mchan->desc; |
|
|
|
/* |
|
* Determine the number of commands that haven't been |
|
* processed (handled by the IRQ handler) yet. |
|
*/ |
|
do { |
|
val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) & |
|
~MDC_CMDS_PROCESSED_INT_ACTIVE; |
|
residue = mdc_chan_readl(mchan, |
|
MDC_ACTIVE_TRANSFER_SIZE); |
|
val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) & |
|
~MDC_CMDS_PROCESSED_INT_ACTIVE; |
|
} while (val1 != val2); |
|
|
|
done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & |
|
MDC_CMDS_PROCESSED_CMDS_DONE_MASK; |
|
processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) & |
|
MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK; |
|
cmds = (done - processed) % |
|
(MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1); |
|
|
|
/* |
|
* If the command loaded event hasn't been processed yet, then |
|
* the difference above includes an extra command. |
|
*/ |
|
if (!mdesc->cmd_loaded) |
|
cmds--; |
|
else |
|
cmds += mdesc->list_cmds_done; |
|
|
|
bytes = mdesc->list_xfer_size; |
|
ldesc = mdesc->list; |
|
for (i = 0; i < cmds; i++) { |
|
bytes -= ldesc->xfer_size + 1; |
|
ldesc = ldesc->next_desc; |
|
} |
|
if (ldesc) { |
|
if (residue != MDC_TRANSFER_SIZE_MASK) |
|
bytes -= ldesc->xfer_size - residue; |
|
else |
|
bytes -= ldesc->xfer_size + 1; |
|
} |
|
} |
|
spin_unlock_irqrestore(&mchan->vc.lock, flags); |
|
|
|
dma_set_residue(txstate, bytes); |
|
|
|
return ret; |
|
} |
|
|
|
static unsigned int mdc_get_new_events(struct mdc_chan *mchan) |
|
{ |
|
u32 val, processed, done1, done2; |
|
unsigned int ret; |
|
|
|
val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); |
|
processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) & |
|
MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK; |
|
/* |
|
* CMDS_DONE may have incremented between reading CMDS_PROCESSED |
|
* and clearing INT_ACTIVE. Re-read CMDS_PROCESSED to ensure we |
|
* didn't miss a command completion. |
|
*/ |
|
do { |
|
val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); |
|
|
|
done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & |
|
MDC_CMDS_PROCESSED_CMDS_DONE_MASK; |
|
|
|
val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK << |
|
MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) | |
|
MDC_CMDS_PROCESSED_INT_ACTIVE); |
|
|
|
val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT; |
|
|
|
mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED); |
|
|
|
val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); |
|
|
|
done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & |
|
MDC_CMDS_PROCESSED_CMDS_DONE_MASK; |
|
} while (done1 != done2); |
|
|
|
if (done1 >= processed) |
|
ret = done1 - processed; |
|
else |
|
ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) - |
|
processed) + done1; |
|
|
|
return ret; |
|
} |
|
|
|
static int mdc_terminate_all(struct dma_chan *chan) |
|
{ |
|
struct mdc_chan *mchan = to_mdc_chan(chan); |
|
unsigned long flags; |
|
LIST_HEAD(head); |
|
|
|
spin_lock_irqsave(&mchan->vc.lock, flags); |
|
|
|
mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL, |
|
MDC_CONTROL_AND_STATUS); |
|
|
|
if (mchan->desc) { |
|
vchan_terminate_vdesc(&mchan->desc->vd); |
|
mchan->desc = NULL; |
|
} |
|
vchan_get_all_descriptors(&mchan->vc, &head); |
|
|
|
mdc_get_new_events(mchan); |
|
|
|
spin_unlock_irqrestore(&mchan->vc.lock, flags); |
|
|
|
vchan_dma_desc_free_list(&mchan->vc, &head); |
|
|
|
return 0; |
|
} |
|
|
|
static void mdc_synchronize(struct dma_chan *chan) |
|
{ |
|
struct mdc_chan *mchan = to_mdc_chan(chan); |
|
|
|
vchan_synchronize(&mchan->vc); |
|
} |
|
|
|
static int mdc_slave_config(struct dma_chan *chan, |
|
struct dma_slave_config *config) |
|
{ |
|
struct mdc_chan *mchan = to_mdc_chan(chan); |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&mchan->vc.lock, flags); |
|
mchan->config = *config; |
|
spin_unlock_irqrestore(&mchan->vc.lock, flags); |
|
|
|
return 0; |
|
} |
|
|
|
static int mdc_alloc_chan_resources(struct dma_chan *chan) |
|
{ |
|
struct mdc_chan *mchan = to_mdc_chan(chan); |
|
struct device *dev = mdma2dev(mchan->mdma); |
|
|
|
return pm_runtime_get_sync(dev); |
|
} |
|
|
|
static void mdc_free_chan_resources(struct dma_chan *chan) |
|
{ |
|
struct mdc_chan *mchan = to_mdc_chan(chan); |
|
struct mdc_dma *mdma = mchan->mdma; |
|
struct device *dev = mdma2dev(mdma); |
|
|
|
mdc_terminate_all(chan); |
|
mdma->soc->disable_chan(mchan); |
|
pm_runtime_put(dev); |
|
} |
|
|
|
static irqreturn_t mdc_chan_irq(int irq, void *dev_id) |
|
{ |
|
struct mdc_chan *mchan = (struct mdc_chan *)dev_id; |
|
struct mdc_tx_desc *mdesc; |
|
unsigned int i, new_events; |
|
|
|
spin_lock(&mchan->vc.lock); |
|
|
|
dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr); |
|
|
|
new_events = mdc_get_new_events(mchan); |
|
|
|
if (!new_events) |
|
goto out; |
|
|
|
mdesc = mchan->desc; |
|
if (!mdesc) { |
|
dev_warn(mdma2dev(mchan->mdma), |
|
"IRQ with no active descriptor on channel %d\n", |
|
mchan->chan_nr); |
|
goto out; |
|
} |
|
|
|
for (i = 0; i < new_events; i++) { |
|
/* |
|
* The first interrupt in a transfer indicates that the |
|
* command list has been loaded, not that a command has |
|
* been completed. |
|
*/ |
|
if (!mdesc->cmd_loaded) { |
|
mdesc->cmd_loaded = true; |
|
continue; |
|
} |
|
|
|
mdesc->list_cmds_done++; |
|
if (mdesc->cyclic) { |
|
mdesc->list_cmds_done %= mdesc->list_len; |
|
if (mdesc->list_cmds_done % mdesc->list_period_len == 0) |
|
vchan_cyclic_callback(&mdesc->vd); |
|
} else if (mdesc->list_cmds_done == mdesc->list_len) { |
|
mchan->desc = NULL; |
|
vchan_cookie_complete(&mdesc->vd); |
|
mdc_issue_desc(mchan); |
|
break; |
|
} |
|
} |
|
out: |
|
spin_unlock(&mchan->vc.lock); |
|
|
|
return IRQ_HANDLED; |
|
} |
|
|
|
static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec, |
|
struct of_dma *ofdma) |
|
{ |
|
struct mdc_dma *mdma = ofdma->of_dma_data; |
|
struct dma_chan *chan; |
|
|
|
if (dma_spec->args_count != 3) |
|
return NULL; |
|
|
|
list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) { |
|
struct mdc_chan *mchan = to_mdc_chan(chan); |
|
|
|
if (!(dma_spec->args[1] & BIT(mchan->chan_nr))) |
|
continue; |
|
if (dma_get_slave_channel(chan)) { |
|
mchan->periph = dma_spec->args[0]; |
|
mchan->thread = dma_spec->args[2]; |
|
return chan; |
|
} |
|
} |
|
|
|
return NULL; |
|
} |
|
|
|
#define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch) (0x120 + 0x4 * ((ch) / 4)) |
|
#define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4)) |
|
#define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK 0x3f |
|
|
|
static void pistachio_mdc_enable_chan(struct mdc_chan *mchan) |
|
{ |
|
struct mdc_dma *mdma = mchan->mdma; |
|
|
|
regmap_update_bits(mdma->periph_regs, |
|
PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr), |
|
PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK << |
|
PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr), |
|
mchan->periph << |
|
PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr)); |
|
} |
|
|
|
static void pistachio_mdc_disable_chan(struct mdc_chan *mchan) |
|
{ |
|
struct mdc_dma *mdma = mchan->mdma; |
|
|
|
regmap_update_bits(mdma->periph_regs, |
|
PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr), |
|
PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK << |
|
PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr), |
|
0); |
|
} |
|
|
|
static const struct mdc_dma_soc_data pistachio_mdc_data = { |
|
.enable_chan = pistachio_mdc_enable_chan, |
|
.disable_chan = pistachio_mdc_disable_chan, |
|
}; |
|
|
|
static const struct of_device_id mdc_dma_of_match[] = { |
|
{ .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, }, |
|
{ }, |
|
}; |
|
MODULE_DEVICE_TABLE(of, mdc_dma_of_match); |
|
|
|
static int img_mdc_runtime_suspend(struct device *dev) |
|
{ |
|
struct mdc_dma *mdma = dev_get_drvdata(dev); |
|
|
|
clk_disable_unprepare(mdma->clk); |
|
|
|
return 0; |
|
} |
|
|
|
static int img_mdc_runtime_resume(struct device *dev) |
|
{ |
|
struct mdc_dma *mdma = dev_get_drvdata(dev); |
|
|
|
return clk_prepare_enable(mdma->clk); |
|
} |
|
|
|
static int mdc_dma_probe(struct platform_device *pdev) |
|
{ |
|
struct mdc_dma *mdma; |
|
struct resource *res; |
|
unsigned int i; |
|
u32 val; |
|
int ret; |
|
|
|
mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL); |
|
if (!mdma) |
|
return -ENOMEM; |
|
platform_set_drvdata(pdev, mdma); |
|
|
|
mdma->soc = of_device_get_match_data(&pdev->dev); |
|
|
|
res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
|
mdma->regs = devm_ioremap_resource(&pdev->dev, res); |
|
if (IS_ERR(mdma->regs)) |
|
return PTR_ERR(mdma->regs); |
|
|
|
mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, |
|
"img,cr-periph"); |
|
if (IS_ERR(mdma->periph_regs)) |
|
return PTR_ERR(mdma->periph_regs); |
|
|
|
mdma->clk = devm_clk_get(&pdev->dev, "sys"); |
|
if (IS_ERR(mdma->clk)) |
|
return PTR_ERR(mdma->clk); |
|
|
|
dma_cap_zero(mdma->dma_dev.cap_mask); |
|
dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask); |
|
dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask); |
|
dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask); |
|
dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask); |
|
|
|
val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A); |
|
mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) & |
|
MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK; |
|
mdma->nr_threads = |
|
1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) & |
|
MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK); |
|
mdma->bus_width = |
|
(1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) & |
|
MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8; |
|
/* |
|
* Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes |
|
* are supported, this makes it possible for the value reported in |
|
* MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size |
|
* of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or |
|
* MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining. To eliminate this |
|
* ambiguity, restrict transfer sizes to one bus-width less than the |
|
* actual maximum. |
|
*/ |
|
mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width; |
|
|
|
of_property_read_u32(pdev->dev.of_node, "dma-channels", |
|
&mdma->nr_channels); |
|
ret = of_property_read_u32(pdev->dev.of_node, |
|
"img,max-burst-multiplier", |
|
&mdma->max_burst_mult); |
|
if (ret) |
|
return ret; |
|
|
|
mdma->dma_dev.dev = &pdev->dev; |
|
mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg; |
|
mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic; |
|
mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy; |
|
mdma->dma_dev.device_alloc_chan_resources = mdc_alloc_chan_resources; |
|
mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources; |
|
mdma->dma_dev.device_tx_status = mdc_tx_status; |
|
mdma->dma_dev.device_issue_pending = mdc_issue_pending; |
|
mdma->dma_dev.device_terminate_all = mdc_terminate_all; |
|
mdma->dma_dev.device_synchronize = mdc_synchronize; |
|
mdma->dma_dev.device_config = mdc_slave_config; |
|
|
|
mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); |
|
mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; |
|
for (i = 1; i <= mdma->bus_width; i <<= 1) { |
|
mdma->dma_dev.src_addr_widths |= BIT(i); |
|
mdma->dma_dev.dst_addr_widths |= BIT(i); |
|
} |
|
|
|
INIT_LIST_HEAD(&mdma->dma_dev.channels); |
|
for (i = 0; i < mdma->nr_channels; i++) { |
|
struct mdc_chan *mchan = &mdma->channels[i]; |
|
|
|
mchan->mdma = mdma; |
|
mchan->chan_nr = i; |
|
mchan->irq = platform_get_irq(pdev, i); |
|
if (mchan->irq < 0) |
|
return mchan->irq; |
|
|
|
ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq, |
|
IRQ_TYPE_LEVEL_HIGH, |
|
dev_name(&pdev->dev), mchan); |
|
if (ret < 0) |
|
return ret; |
|
|
|
mchan->vc.desc_free = mdc_desc_free; |
|
vchan_init(&mchan->vc, &mdma->dma_dev); |
|
} |
|
|
|
mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev, |
|
sizeof(struct mdc_hw_list_desc), |
|
4, 0); |
|
if (!mdma->desc_pool) |
|
return -ENOMEM; |
|
|
|
pm_runtime_enable(&pdev->dev); |
|
if (!pm_runtime_enabled(&pdev->dev)) { |
|
ret = img_mdc_runtime_resume(&pdev->dev); |
|
if (ret) |
|
return ret; |
|
} |
|
|
|
ret = dma_async_device_register(&mdma->dma_dev); |
|
if (ret) |
|
goto suspend; |
|
|
|
ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma); |
|
if (ret) |
|
goto unregister; |
|
|
|
dev_info(&pdev->dev, "MDC with %u channels and %u threads\n", |
|
mdma->nr_channels, mdma->nr_threads); |
|
|
|
return 0; |
|
|
|
unregister: |
|
dma_async_device_unregister(&mdma->dma_dev); |
|
suspend: |
|
if (!pm_runtime_enabled(&pdev->dev)) |
|
img_mdc_runtime_suspend(&pdev->dev); |
|
pm_runtime_disable(&pdev->dev); |
|
return ret; |
|
} |
|
|
|
static int mdc_dma_remove(struct platform_device *pdev) |
|
{ |
|
struct mdc_dma *mdma = platform_get_drvdata(pdev); |
|
struct mdc_chan *mchan, *next; |
|
|
|
of_dma_controller_free(pdev->dev.of_node); |
|
dma_async_device_unregister(&mdma->dma_dev); |
|
|
|
list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels, |
|
vc.chan.device_node) { |
|
list_del(&mchan->vc.chan.device_node); |
|
|
|
devm_free_irq(&pdev->dev, mchan->irq, mchan); |
|
|
|
tasklet_kill(&mchan->vc.task); |
|
} |
|
|
|
pm_runtime_disable(&pdev->dev); |
|
if (!pm_runtime_status_suspended(&pdev->dev)) |
|
img_mdc_runtime_suspend(&pdev->dev); |
|
|
|
return 0; |
|
} |
|
|
|
#ifdef CONFIG_PM_SLEEP |
|
static int img_mdc_suspend_late(struct device *dev) |
|
{ |
|
struct mdc_dma *mdma = dev_get_drvdata(dev); |
|
int i; |
|
|
|
/* Check that all channels are idle */ |
|
for (i = 0; i < mdma->nr_channels; i++) { |
|
struct mdc_chan *mchan = &mdma->channels[i]; |
|
|
|
if (unlikely(mchan->desc)) |
|
return -EBUSY; |
|
} |
|
|
|
return pm_runtime_force_suspend(dev); |
|
} |
|
|
|
static int img_mdc_resume_early(struct device *dev) |
|
{ |
|
return pm_runtime_force_resume(dev); |
|
} |
|
#endif /* CONFIG_PM_SLEEP */ |
|
|
|
static const struct dev_pm_ops img_mdc_pm_ops = { |
|
SET_RUNTIME_PM_OPS(img_mdc_runtime_suspend, |
|
img_mdc_runtime_resume, NULL) |
|
SET_LATE_SYSTEM_SLEEP_PM_OPS(img_mdc_suspend_late, |
|
img_mdc_resume_early) |
|
}; |
|
|
|
static struct platform_driver mdc_dma_driver = { |
|
.driver = { |
|
.name = "img-mdc-dma", |
|
.pm = &img_mdc_pm_ops, |
|
.of_match_table = of_match_ptr(mdc_dma_of_match), |
|
}, |
|
.probe = mdc_dma_probe, |
|
.remove = mdc_dma_remove, |
|
}; |
|
module_platform_driver(mdc_dma_driver); |
|
|
|
MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver"); |
|
MODULE_AUTHOR("Andrew Bresticker <[email protected]>"); |
|
MODULE_LICENSE("GPL v2");
|
|
|