forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1434 lines
39 KiB
1434 lines
39 KiB
// SPDX-License-Identifier: GPL-2.0-or-later |
|
/* |
|
* Driver for the Cirrus Logic EP93xx DMA Controller |
|
* |
|
* Copyright (C) 2011 Mika Westerberg |
|
* |
|
* DMA M2P implementation is based on the original |
|
* arch/arm/mach-ep93xx/dma-m2p.c which has following copyrights: |
|
* |
|
* Copyright (C) 2006 Lennert Buytenhek <[email protected]> |
|
* Copyright (C) 2006 Applied Data Systems |
|
* Copyright (C) 2009 Ryan Mallon <[email protected]> |
|
* |
|
* This driver is based on dw_dmac and amba-pl08x drivers. |
|
*/ |
|
|
|
#include <linux/clk.h> |
|
#include <linux/init.h> |
|
#include <linux/interrupt.h> |
|
#include <linux/dmaengine.h> |
|
#include <linux/module.h> |
|
#include <linux/mod_devicetable.h> |
|
#include <linux/platform_device.h> |
|
#include <linux/slab.h> |
|
|
|
#include <linux/platform_data/dma-ep93xx.h> |
|
|
|
#include "dmaengine.h" |
|
|
|
/* M2P registers */ |
|
#define M2P_CONTROL 0x0000 |
|
#define M2P_CONTROL_STALLINT BIT(0) |
|
#define M2P_CONTROL_NFBINT BIT(1) |
|
#define M2P_CONTROL_CH_ERROR_INT BIT(3) |
|
#define M2P_CONTROL_ENABLE BIT(4) |
|
#define M2P_CONTROL_ICE BIT(6) |
|
|
|
#define M2P_INTERRUPT 0x0004 |
|
#define M2P_INTERRUPT_STALL BIT(0) |
|
#define M2P_INTERRUPT_NFB BIT(1) |
|
#define M2P_INTERRUPT_ERROR BIT(3) |
|
|
|
#define M2P_PPALLOC 0x0008 |
|
#define M2P_STATUS 0x000c |
|
|
|
#define M2P_MAXCNT0 0x0020 |
|
#define M2P_BASE0 0x0024 |
|
#define M2P_MAXCNT1 0x0030 |
|
#define M2P_BASE1 0x0034 |
|
|
|
#define M2P_STATE_IDLE 0 |
|
#define M2P_STATE_STALL 1 |
|
#define M2P_STATE_ON 2 |
|
#define M2P_STATE_NEXT 3 |
|
|
|
/* M2M registers */ |
|
#define M2M_CONTROL 0x0000 |
|
#define M2M_CONTROL_DONEINT BIT(2) |
|
#define M2M_CONTROL_ENABLE BIT(3) |
|
#define M2M_CONTROL_START BIT(4) |
|
#define M2M_CONTROL_DAH BIT(11) |
|
#define M2M_CONTROL_SAH BIT(12) |
|
#define M2M_CONTROL_PW_SHIFT 9 |
|
#define M2M_CONTROL_PW_8 (0 << M2M_CONTROL_PW_SHIFT) |
|
#define M2M_CONTROL_PW_16 (1 << M2M_CONTROL_PW_SHIFT) |
|
#define M2M_CONTROL_PW_32 (2 << M2M_CONTROL_PW_SHIFT) |
|
#define M2M_CONTROL_PW_MASK (3 << M2M_CONTROL_PW_SHIFT) |
|
#define M2M_CONTROL_TM_SHIFT 13 |
|
#define M2M_CONTROL_TM_TX (1 << M2M_CONTROL_TM_SHIFT) |
|
#define M2M_CONTROL_TM_RX (2 << M2M_CONTROL_TM_SHIFT) |
|
#define M2M_CONTROL_NFBINT BIT(21) |
|
#define M2M_CONTROL_RSS_SHIFT 22 |
|
#define M2M_CONTROL_RSS_SSPRX (1 << M2M_CONTROL_RSS_SHIFT) |
|
#define M2M_CONTROL_RSS_SSPTX (2 << M2M_CONTROL_RSS_SHIFT) |
|
#define M2M_CONTROL_RSS_IDE (3 << M2M_CONTROL_RSS_SHIFT) |
|
#define M2M_CONTROL_NO_HDSK BIT(24) |
|
#define M2M_CONTROL_PWSC_SHIFT 25 |
|
|
|
#define M2M_INTERRUPT 0x0004 |
|
#define M2M_INTERRUPT_MASK 6 |
|
|
|
#define M2M_STATUS 0x000c |
|
#define M2M_STATUS_CTL_SHIFT 1 |
|
#define M2M_STATUS_CTL_IDLE (0 << M2M_STATUS_CTL_SHIFT) |
|
#define M2M_STATUS_CTL_STALL (1 << M2M_STATUS_CTL_SHIFT) |
|
#define M2M_STATUS_CTL_MEMRD (2 << M2M_STATUS_CTL_SHIFT) |
|
#define M2M_STATUS_CTL_MEMWR (3 << M2M_STATUS_CTL_SHIFT) |
|
#define M2M_STATUS_CTL_BWCWAIT (4 << M2M_STATUS_CTL_SHIFT) |
|
#define M2M_STATUS_CTL_MASK (7 << M2M_STATUS_CTL_SHIFT) |
|
#define M2M_STATUS_BUF_SHIFT 4 |
|
#define M2M_STATUS_BUF_NO (0 << M2M_STATUS_BUF_SHIFT) |
|
#define M2M_STATUS_BUF_ON (1 << M2M_STATUS_BUF_SHIFT) |
|
#define M2M_STATUS_BUF_NEXT (2 << M2M_STATUS_BUF_SHIFT) |
|
#define M2M_STATUS_BUF_MASK (3 << M2M_STATUS_BUF_SHIFT) |
|
#define M2M_STATUS_DONE BIT(6) |
|
|
|
#define M2M_BCR0 0x0010 |
|
#define M2M_BCR1 0x0014 |
|
#define M2M_SAR_BASE0 0x0018 |
|
#define M2M_SAR_BASE1 0x001c |
|
#define M2M_DAR_BASE0 0x002c |
|
#define M2M_DAR_BASE1 0x0030 |
|
|
|
#define DMA_MAX_CHAN_BYTES 0xffff |
|
#define DMA_MAX_CHAN_DESCRIPTORS 32 |
|
|
|
struct ep93xx_dma_engine; |
|
static int ep93xx_dma_slave_config_write(struct dma_chan *chan, |
|
enum dma_transfer_direction dir, |
|
struct dma_slave_config *config); |
|
|
|
/** |
|
* struct ep93xx_dma_desc - EP93xx specific transaction descriptor |
|
* @src_addr: source address of the transaction |
|
* @dst_addr: destination address of the transaction |
|
* @size: size of the transaction (in bytes) |
|
* @complete: this descriptor is completed |
|
* @txd: dmaengine API descriptor |
|
* @tx_list: list of linked descriptors |
|
* @node: link used for putting this into a channel queue |
|
*/ |
|
struct ep93xx_dma_desc { |
|
u32 src_addr; |
|
u32 dst_addr; |
|
size_t size; |
|
bool complete; |
|
struct dma_async_tx_descriptor txd; |
|
struct list_head tx_list; |
|
struct list_head node; |
|
}; |
|
|
|
/** |
|
* struct ep93xx_dma_chan - an EP93xx DMA M2P/M2M channel |
|
* @chan: dmaengine API channel |
|
* @edma: pointer to to the engine device |
|
* @regs: memory mapped registers |
|
* @irq: interrupt number of the channel |
|
* @clk: clock used by this channel |
|
* @tasklet: channel specific tasklet used for callbacks |
|
* @lock: lock protecting the fields following |
|
* @flags: flags for the channel |
|
* @buffer: which buffer to use next (0/1) |
|
* @active: flattened chain of descriptors currently being processed |
|
* @queue: pending descriptors which are handled next |
|
* @free_list: list of free descriptors which can be used |
|
* @runtime_addr: physical address currently used as dest/src (M2M only). This |
|
* is set via .device_config before slave operation is |
|
* prepared |
|
* @runtime_ctrl: M2M runtime values for the control register. |
|
* @slave_config: slave configuration |
|
* |
|
* As EP93xx DMA controller doesn't support real chained DMA descriptors we |
|
* will have slightly different scheme here: @active points to a head of |
|
* flattened DMA descriptor chain. |
|
* |
|
* @queue holds pending transactions. These are linked through the first |
|
* descriptor in the chain. When a descriptor is moved to the @active queue, |
|
* the first and chained descriptors are flattened into a single list. |
|
* |
|
* @chan.private holds pointer to &struct ep93xx_dma_data which contains |
|
* necessary channel configuration information. For memcpy channels this must |
|
* be %NULL. |
|
*/ |
|
struct ep93xx_dma_chan { |
|
struct dma_chan chan; |
|
const struct ep93xx_dma_engine *edma; |
|
void __iomem *regs; |
|
int irq; |
|
struct clk *clk; |
|
struct tasklet_struct tasklet; |
|
/* protects the fields following */ |
|
spinlock_t lock; |
|
unsigned long flags; |
|
/* Channel is configured for cyclic transfers */ |
|
#define EP93XX_DMA_IS_CYCLIC 0 |
|
|
|
int buffer; |
|
struct list_head active; |
|
struct list_head queue; |
|
struct list_head free_list; |
|
u32 runtime_addr; |
|
u32 runtime_ctrl; |
|
struct dma_slave_config slave_config; |
|
}; |
|
|
|
/** |
|
* struct ep93xx_dma_engine - the EP93xx DMA engine instance |
|
* @dma_dev: holds the dmaengine device |
|
* @m2m: is this an M2M or M2P device |
|
* @hw_setup: method which sets the channel up for operation |
|
* @hw_synchronize: synchronizes DMA channel termination to current context |
|
* @hw_shutdown: shuts the channel down and flushes whatever is left |
|
* @hw_submit: pushes active descriptor(s) to the hardware |
|
* @hw_interrupt: handle the interrupt |
|
* @num_channels: number of channels for this instance |
|
* @channels: array of channels |
|
* |
|
* There is one instance of this struct for the M2P channels and one for the |
|
* M2M channels. hw_xxx() methods are used to perform operations which are |
|
* different on M2M and M2P channels. These methods are called with channel |
|
* lock held and interrupts disabled so they cannot sleep. |
|
*/ |
|
struct ep93xx_dma_engine { |
|
struct dma_device dma_dev; |
|
bool m2m; |
|
int (*hw_setup)(struct ep93xx_dma_chan *); |
|
void (*hw_synchronize)(struct ep93xx_dma_chan *); |
|
void (*hw_shutdown)(struct ep93xx_dma_chan *); |
|
void (*hw_submit)(struct ep93xx_dma_chan *); |
|
int (*hw_interrupt)(struct ep93xx_dma_chan *); |
|
#define INTERRUPT_UNKNOWN 0 |
|
#define INTERRUPT_DONE 1 |
|
#define INTERRUPT_NEXT_BUFFER 2 |
|
|
|
size_t num_channels; |
|
struct ep93xx_dma_chan channels[]; |
|
}; |
|
|
|
static inline struct device *chan2dev(struct ep93xx_dma_chan *edmac) |
|
{ |
|
return &edmac->chan.dev->device; |
|
} |
|
|
|
static struct ep93xx_dma_chan *to_ep93xx_dma_chan(struct dma_chan *chan) |
|
{ |
|
return container_of(chan, struct ep93xx_dma_chan, chan); |
|
} |
|
|
|
/** |
|
* ep93xx_dma_set_active - set new active descriptor chain |
|
* @edmac: channel |
|
* @desc: head of the new active descriptor chain |
|
* |
|
* Sets @desc to be the head of the new active descriptor chain. This is the |
|
* chain which is processed next. The active list must be empty before calling |
|
* this function. |
|
* |
|
* Called with @edmac->lock held and interrupts disabled. |
|
*/ |
|
static void ep93xx_dma_set_active(struct ep93xx_dma_chan *edmac, |
|
struct ep93xx_dma_desc *desc) |
|
{ |
|
BUG_ON(!list_empty(&edmac->active)); |
|
|
|
list_add_tail(&desc->node, &edmac->active); |
|
|
|
/* Flatten the @desc->tx_list chain into @edmac->active list */ |
|
while (!list_empty(&desc->tx_list)) { |
|
struct ep93xx_dma_desc *d = list_first_entry(&desc->tx_list, |
|
struct ep93xx_dma_desc, node); |
|
|
|
/* |
|
* We copy the callback parameters from the first descriptor |
|
* to all the chained descriptors. This way we can call the |
|
* callback without having to find out the first descriptor in |
|
* the chain. Useful for cyclic transfers. |
|
*/ |
|
d->txd.callback = desc->txd.callback; |
|
d->txd.callback_param = desc->txd.callback_param; |
|
|
|
list_move_tail(&d->node, &edmac->active); |
|
} |
|
} |
|
|
|
/* Called with @edmac->lock held and interrupts disabled */ |
|
static struct ep93xx_dma_desc * |
|
ep93xx_dma_get_active(struct ep93xx_dma_chan *edmac) |
|
{ |
|
return list_first_entry_or_null(&edmac->active, |
|
struct ep93xx_dma_desc, node); |
|
} |
|
|
|
/** |
|
* ep93xx_dma_advance_active - advances to the next active descriptor |
|
* @edmac: channel |
|
* |
|
* Function advances active descriptor to the next in the @edmac->active and |
|
* returns %true if we still have descriptors in the chain to process. |
|
* Otherwise returns %false. |
|
* |
|
* When the channel is in cyclic mode always returns %true. |
|
* |
|
* Called with @edmac->lock held and interrupts disabled. |
|
*/ |
|
static bool ep93xx_dma_advance_active(struct ep93xx_dma_chan *edmac) |
|
{ |
|
struct ep93xx_dma_desc *desc; |
|
|
|
list_rotate_left(&edmac->active); |
|
|
|
if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) |
|
return true; |
|
|
|
desc = ep93xx_dma_get_active(edmac); |
|
if (!desc) |
|
return false; |
|
|
|
/* |
|
* If txd.cookie is set it means that we are back in the first |
|
* descriptor in the chain and hence done with it. |
|
*/ |
|
return !desc->txd.cookie; |
|
} |
|
|
|
/* |
|
* M2P DMA implementation |
|
*/ |
|
|
|
static void m2p_set_control(struct ep93xx_dma_chan *edmac, u32 control) |
|
{ |
|
writel(control, edmac->regs + M2P_CONTROL); |
|
/* |
|
* EP93xx User's Guide states that we must perform a dummy read after |
|
* write to the control register. |
|
*/ |
|
readl(edmac->regs + M2P_CONTROL); |
|
} |
|
|
|
static int m2p_hw_setup(struct ep93xx_dma_chan *edmac) |
|
{ |
|
struct ep93xx_dma_data *data = edmac->chan.private; |
|
u32 control; |
|
|
|
writel(data->port & 0xf, edmac->regs + M2P_PPALLOC); |
|
|
|
control = M2P_CONTROL_CH_ERROR_INT | M2P_CONTROL_ICE |
|
| M2P_CONTROL_ENABLE; |
|
m2p_set_control(edmac, control); |
|
|
|
edmac->buffer = 0; |
|
|
|
return 0; |
|
} |
|
|
|
static inline u32 m2p_channel_state(struct ep93xx_dma_chan *edmac) |
|
{ |
|
return (readl(edmac->regs + M2P_STATUS) >> 4) & 0x3; |
|
} |
|
|
|
static void m2p_hw_synchronize(struct ep93xx_dma_chan *edmac) |
|
{ |
|
unsigned long flags; |
|
u32 control; |
|
|
|
spin_lock_irqsave(&edmac->lock, flags); |
|
control = readl(edmac->regs + M2P_CONTROL); |
|
control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT); |
|
m2p_set_control(edmac, control); |
|
spin_unlock_irqrestore(&edmac->lock, flags); |
|
|
|
while (m2p_channel_state(edmac) >= M2P_STATE_ON) |
|
schedule(); |
|
} |
|
|
|
static void m2p_hw_shutdown(struct ep93xx_dma_chan *edmac) |
|
{ |
|
m2p_set_control(edmac, 0); |
|
|
|
while (m2p_channel_state(edmac) != M2P_STATE_IDLE) |
|
dev_warn(chan2dev(edmac), "M2P: Not yet IDLE\n"); |
|
} |
|
|
|
static void m2p_fill_desc(struct ep93xx_dma_chan *edmac) |
|
{ |
|
struct ep93xx_dma_desc *desc; |
|
u32 bus_addr; |
|
|
|
desc = ep93xx_dma_get_active(edmac); |
|
if (!desc) { |
|
dev_warn(chan2dev(edmac), "M2P: empty descriptor list\n"); |
|
return; |
|
} |
|
|
|
if (ep93xx_dma_chan_direction(&edmac->chan) == DMA_MEM_TO_DEV) |
|
bus_addr = desc->src_addr; |
|
else |
|
bus_addr = desc->dst_addr; |
|
|
|
if (edmac->buffer == 0) { |
|
writel(desc->size, edmac->regs + M2P_MAXCNT0); |
|
writel(bus_addr, edmac->regs + M2P_BASE0); |
|
} else { |
|
writel(desc->size, edmac->regs + M2P_MAXCNT1); |
|
writel(bus_addr, edmac->regs + M2P_BASE1); |
|
} |
|
|
|
edmac->buffer ^= 1; |
|
} |
|
|
|
static void m2p_hw_submit(struct ep93xx_dma_chan *edmac) |
|
{ |
|
u32 control = readl(edmac->regs + M2P_CONTROL); |
|
|
|
m2p_fill_desc(edmac); |
|
control |= M2P_CONTROL_STALLINT; |
|
|
|
if (ep93xx_dma_advance_active(edmac)) { |
|
m2p_fill_desc(edmac); |
|
control |= M2P_CONTROL_NFBINT; |
|
} |
|
|
|
m2p_set_control(edmac, control); |
|
} |
|
|
|
static int m2p_hw_interrupt(struct ep93xx_dma_chan *edmac) |
|
{ |
|
u32 irq_status = readl(edmac->regs + M2P_INTERRUPT); |
|
u32 control; |
|
|
|
if (irq_status & M2P_INTERRUPT_ERROR) { |
|
struct ep93xx_dma_desc *desc = ep93xx_dma_get_active(edmac); |
|
|
|
/* Clear the error interrupt */ |
|
writel(1, edmac->regs + M2P_INTERRUPT); |
|
|
|
/* |
|
* It seems that there is no easy way of reporting errors back |
|
* to client so we just report the error here and continue as |
|
* usual. |
|
* |
|
* Revisit this when there is a mechanism to report back the |
|
* errors. |
|
*/ |
|
dev_err(chan2dev(edmac), |
|
"DMA transfer failed! Details:\n" |
|
"\tcookie : %d\n" |
|
"\tsrc_addr : 0x%08x\n" |
|
"\tdst_addr : 0x%08x\n" |
|
"\tsize : %zu\n", |
|
desc->txd.cookie, desc->src_addr, desc->dst_addr, |
|
desc->size); |
|
} |
|
|
|
/* |
|
* Even latest E2 silicon revision sometimes assert STALL interrupt |
|
* instead of NFB. Therefore we treat them equally, basing on the |
|
* amount of data we still have to transfer. |
|
*/ |
|
if (!(irq_status & (M2P_INTERRUPT_STALL | M2P_INTERRUPT_NFB))) |
|
return INTERRUPT_UNKNOWN; |
|
|
|
if (ep93xx_dma_advance_active(edmac)) { |
|
m2p_fill_desc(edmac); |
|
return INTERRUPT_NEXT_BUFFER; |
|
} |
|
|
|
/* Disable interrupts */ |
|
control = readl(edmac->regs + M2P_CONTROL); |
|
control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT); |
|
m2p_set_control(edmac, control); |
|
|
|
return INTERRUPT_DONE; |
|
} |
|
|
|
/* |
|
* M2M DMA implementation |
|
*/ |
|
|
|
static int m2m_hw_setup(struct ep93xx_dma_chan *edmac) |
|
{ |
|
const struct ep93xx_dma_data *data = edmac->chan.private; |
|
u32 control = 0; |
|
|
|
if (!data) { |
|
/* This is memcpy channel, nothing to configure */ |
|
writel(control, edmac->regs + M2M_CONTROL); |
|
return 0; |
|
} |
|
|
|
switch (data->port) { |
|
case EP93XX_DMA_SSP: |
|
/* |
|
* This was found via experimenting - anything less than 5 |
|
* causes the channel to perform only a partial transfer which |
|
* leads to problems since we don't get DONE interrupt then. |
|
*/ |
|
control = (5 << M2M_CONTROL_PWSC_SHIFT); |
|
control |= M2M_CONTROL_NO_HDSK; |
|
|
|
if (data->direction == DMA_MEM_TO_DEV) { |
|
control |= M2M_CONTROL_DAH; |
|
control |= M2M_CONTROL_TM_TX; |
|
control |= M2M_CONTROL_RSS_SSPTX; |
|
} else { |
|
control |= M2M_CONTROL_SAH; |
|
control |= M2M_CONTROL_TM_RX; |
|
control |= M2M_CONTROL_RSS_SSPRX; |
|
} |
|
break; |
|
|
|
case EP93XX_DMA_IDE: |
|
/* |
|
* This IDE part is totally untested. Values below are taken |
|
* from the EP93xx Users's Guide and might not be correct. |
|
*/ |
|
if (data->direction == DMA_MEM_TO_DEV) { |
|
/* Worst case from the UG */ |
|
control = (3 << M2M_CONTROL_PWSC_SHIFT); |
|
control |= M2M_CONTROL_DAH; |
|
control |= M2M_CONTROL_TM_TX; |
|
} else { |
|
control = (2 << M2M_CONTROL_PWSC_SHIFT); |
|
control |= M2M_CONTROL_SAH; |
|
control |= M2M_CONTROL_TM_RX; |
|
} |
|
|
|
control |= M2M_CONTROL_NO_HDSK; |
|
control |= M2M_CONTROL_RSS_IDE; |
|
control |= M2M_CONTROL_PW_16; |
|
break; |
|
|
|
default: |
|
return -EINVAL; |
|
} |
|
|
|
writel(control, edmac->regs + M2M_CONTROL); |
|
return 0; |
|
} |
|
|
|
static void m2m_hw_shutdown(struct ep93xx_dma_chan *edmac) |
|
{ |
|
/* Just disable the channel */ |
|
writel(0, edmac->regs + M2M_CONTROL); |
|
} |
|
|
|
static void m2m_fill_desc(struct ep93xx_dma_chan *edmac) |
|
{ |
|
struct ep93xx_dma_desc *desc; |
|
|
|
desc = ep93xx_dma_get_active(edmac); |
|
if (!desc) { |
|
dev_warn(chan2dev(edmac), "M2M: empty descriptor list\n"); |
|
return; |
|
} |
|
|
|
if (edmac->buffer == 0) { |
|
writel(desc->src_addr, edmac->regs + M2M_SAR_BASE0); |
|
writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE0); |
|
writel(desc->size, edmac->regs + M2M_BCR0); |
|
} else { |
|
writel(desc->src_addr, edmac->regs + M2M_SAR_BASE1); |
|
writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE1); |
|
writel(desc->size, edmac->regs + M2M_BCR1); |
|
} |
|
|
|
edmac->buffer ^= 1; |
|
} |
|
|
|
static void m2m_hw_submit(struct ep93xx_dma_chan *edmac) |
|
{ |
|
struct ep93xx_dma_data *data = edmac->chan.private; |
|
u32 control = readl(edmac->regs + M2M_CONTROL); |
|
|
|
/* |
|
* Since we allow clients to configure PW (peripheral width) we always |
|
* clear PW bits here and then set them according what is given in |
|
* the runtime configuration. |
|
*/ |
|
control &= ~M2M_CONTROL_PW_MASK; |
|
control |= edmac->runtime_ctrl; |
|
|
|
m2m_fill_desc(edmac); |
|
control |= M2M_CONTROL_DONEINT; |
|
|
|
if (ep93xx_dma_advance_active(edmac)) { |
|
m2m_fill_desc(edmac); |
|
control |= M2M_CONTROL_NFBINT; |
|
} |
|
|
|
/* |
|
* Now we can finally enable the channel. For M2M channel this must be |
|
* done _after_ the BCRx registers are programmed. |
|
*/ |
|
control |= M2M_CONTROL_ENABLE; |
|
writel(control, edmac->regs + M2M_CONTROL); |
|
|
|
if (!data) { |
|
/* |
|
* For memcpy channels the software trigger must be asserted |
|
* in order to start the memcpy operation. |
|
*/ |
|
control |= M2M_CONTROL_START; |
|
writel(control, edmac->regs + M2M_CONTROL); |
|
} |
|
} |
|
|
|
/* |
|
* According to EP93xx User's Guide, we should receive DONE interrupt when all |
|
* M2M DMA controller transactions complete normally. This is not always the |
|
* case - sometimes EP93xx M2M DMA asserts DONE interrupt when the DMA channel |
|
* is still running (channel Buffer FSM in DMA_BUF_ON state, and channel |
|
* Control FSM in DMA_MEM_RD state, observed at least in IDE-DMA operation). |
|
* In effect, disabling the channel when only DONE bit is set could stop |
|
* currently running DMA transfer. To avoid this, we use Buffer FSM and |
|
* Control FSM to check current state of DMA channel. |
|
*/ |
|
static int m2m_hw_interrupt(struct ep93xx_dma_chan *edmac) |
|
{ |
|
u32 status = readl(edmac->regs + M2M_STATUS); |
|
u32 ctl_fsm = status & M2M_STATUS_CTL_MASK; |
|
u32 buf_fsm = status & M2M_STATUS_BUF_MASK; |
|
bool done = status & M2M_STATUS_DONE; |
|
bool last_done; |
|
u32 control; |
|
struct ep93xx_dma_desc *desc; |
|
|
|
/* Accept only DONE and NFB interrupts */ |
|
if (!(readl(edmac->regs + M2M_INTERRUPT) & M2M_INTERRUPT_MASK)) |
|
return INTERRUPT_UNKNOWN; |
|
|
|
if (done) { |
|
/* Clear the DONE bit */ |
|
writel(0, edmac->regs + M2M_INTERRUPT); |
|
} |
|
|
|
/* |
|
* Check whether we are done with descriptors or not. This, together |
|
* with DMA channel state, determines action to take in interrupt. |
|
*/ |
|
desc = ep93xx_dma_get_active(edmac); |
|
last_done = !desc || desc->txd.cookie; |
|
|
|
/* |
|
* Use M2M DMA Buffer FSM and Control FSM to check current state of |
|
* DMA channel. Using DONE and NFB bits from channel status register |
|
* or bits from channel interrupt register is not reliable. |
|
*/ |
|
if (!last_done && |
|
(buf_fsm == M2M_STATUS_BUF_NO || |
|
buf_fsm == M2M_STATUS_BUF_ON)) { |
|
/* |
|
* Two buffers are ready for update when Buffer FSM is in |
|
* DMA_NO_BUF state. Only one buffer can be prepared without |
|
* disabling the channel or polling the DONE bit. |
|
* To simplify things, always prepare only one buffer. |
|
*/ |
|
if (ep93xx_dma_advance_active(edmac)) { |
|
m2m_fill_desc(edmac); |
|
if (done && !edmac->chan.private) { |
|
/* Software trigger for memcpy channel */ |
|
control = readl(edmac->regs + M2M_CONTROL); |
|
control |= M2M_CONTROL_START; |
|
writel(control, edmac->regs + M2M_CONTROL); |
|
} |
|
return INTERRUPT_NEXT_BUFFER; |
|
} else { |
|
last_done = true; |
|
} |
|
} |
|
|
|
/* |
|
* Disable the channel only when Buffer FSM is in DMA_NO_BUF state |
|
* and Control FSM is in DMA_STALL state. |
|
*/ |
|
if (last_done && |
|
buf_fsm == M2M_STATUS_BUF_NO && |
|
ctl_fsm == M2M_STATUS_CTL_STALL) { |
|
/* Disable interrupts and the channel */ |
|
control = readl(edmac->regs + M2M_CONTROL); |
|
control &= ~(M2M_CONTROL_DONEINT | M2M_CONTROL_NFBINT |
|
| M2M_CONTROL_ENABLE); |
|
writel(control, edmac->regs + M2M_CONTROL); |
|
return INTERRUPT_DONE; |
|
} |
|
|
|
/* |
|
* Nothing to do this time. |
|
*/ |
|
return INTERRUPT_NEXT_BUFFER; |
|
} |
|
|
|
/* |
|
* DMA engine API implementation |
|
*/ |
|
|
|
static struct ep93xx_dma_desc * |
|
ep93xx_dma_desc_get(struct ep93xx_dma_chan *edmac) |
|
{ |
|
struct ep93xx_dma_desc *desc, *_desc; |
|
struct ep93xx_dma_desc *ret = NULL; |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&edmac->lock, flags); |
|
list_for_each_entry_safe(desc, _desc, &edmac->free_list, node) { |
|
if (async_tx_test_ack(&desc->txd)) { |
|
list_del_init(&desc->node); |
|
|
|
/* Re-initialize the descriptor */ |
|
desc->src_addr = 0; |
|
desc->dst_addr = 0; |
|
desc->size = 0; |
|
desc->complete = false; |
|
desc->txd.cookie = 0; |
|
desc->txd.callback = NULL; |
|
desc->txd.callback_param = NULL; |
|
|
|
ret = desc; |
|
break; |
|
} |
|
} |
|
spin_unlock_irqrestore(&edmac->lock, flags); |
|
return ret; |
|
} |
|
|
|
static void ep93xx_dma_desc_put(struct ep93xx_dma_chan *edmac, |
|
struct ep93xx_dma_desc *desc) |
|
{ |
|
if (desc) { |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&edmac->lock, flags); |
|
list_splice_init(&desc->tx_list, &edmac->free_list); |
|
list_add(&desc->node, &edmac->free_list); |
|
spin_unlock_irqrestore(&edmac->lock, flags); |
|
} |
|
} |
|
|
|
/** |
|
* ep93xx_dma_advance_work - start processing the next pending transaction |
|
* @edmac: channel |
|
* |
|
* If we have pending transactions queued and we are currently idling, this |
|
* function takes the next queued transaction from the @edmac->queue and |
|
* pushes it to the hardware for execution. |
|
*/ |
|
static void ep93xx_dma_advance_work(struct ep93xx_dma_chan *edmac) |
|
{ |
|
struct ep93xx_dma_desc *new; |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&edmac->lock, flags); |
|
if (!list_empty(&edmac->active) || list_empty(&edmac->queue)) { |
|
spin_unlock_irqrestore(&edmac->lock, flags); |
|
return; |
|
} |
|
|
|
/* Take the next descriptor from the pending queue */ |
|
new = list_first_entry(&edmac->queue, struct ep93xx_dma_desc, node); |
|
list_del_init(&new->node); |
|
|
|
ep93xx_dma_set_active(edmac, new); |
|
|
|
/* Push it to the hardware */ |
|
edmac->edma->hw_submit(edmac); |
|
spin_unlock_irqrestore(&edmac->lock, flags); |
|
} |
|
|
|
static void ep93xx_dma_tasklet(struct tasklet_struct *t) |
|
{ |
|
struct ep93xx_dma_chan *edmac = from_tasklet(edmac, t, tasklet); |
|
struct ep93xx_dma_desc *desc, *d; |
|
struct dmaengine_desc_callback cb; |
|
LIST_HEAD(list); |
|
|
|
memset(&cb, 0, sizeof(cb)); |
|
spin_lock_irq(&edmac->lock); |
|
/* |
|
* If dma_terminate_all() was called before we get to run, the active |
|
* list has become empty. If that happens we aren't supposed to do |
|
* anything more than call ep93xx_dma_advance_work(). |
|
*/ |
|
desc = ep93xx_dma_get_active(edmac); |
|
if (desc) { |
|
if (desc->complete) { |
|
/* mark descriptor complete for non cyclic case only */ |
|
if (!test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) |
|
dma_cookie_complete(&desc->txd); |
|
list_splice_init(&edmac->active, &list); |
|
} |
|
dmaengine_desc_get_callback(&desc->txd, &cb); |
|
} |
|
spin_unlock_irq(&edmac->lock); |
|
|
|
/* Pick up the next descriptor from the queue */ |
|
ep93xx_dma_advance_work(edmac); |
|
|
|
/* Now we can release all the chained descriptors */ |
|
list_for_each_entry_safe(desc, d, &list, node) { |
|
dma_descriptor_unmap(&desc->txd); |
|
ep93xx_dma_desc_put(edmac, desc); |
|
} |
|
|
|
dmaengine_desc_callback_invoke(&cb, NULL); |
|
} |
|
|
|
static irqreturn_t ep93xx_dma_interrupt(int irq, void *dev_id) |
|
{ |
|
struct ep93xx_dma_chan *edmac = dev_id; |
|
struct ep93xx_dma_desc *desc; |
|
irqreturn_t ret = IRQ_HANDLED; |
|
|
|
spin_lock(&edmac->lock); |
|
|
|
desc = ep93xx_dma_get_active(edmac); |
|
if (!desc) { |
|
dev_warn(chan2dev(edmac), |
|
"got interrupt while active list is empty\n"); |
|
spin_unlock(&edmac->lock); |
|
return IRQ_NONE; |
|
} |
|
|
|
switch (edmac->edma->hw_interrupt(edmac)) { |
|
case INTERRUPT_DONE: |
|
desc->complete = true; |
|
tasklet_schedule(&edmac->tasklet); |
|
break; |
|
|
|
case INTERRUPT_NEXT_BUFFER: |
|
if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) |
|
tasklet_schedule(&edmac->tasklet); |
|
break; |
|
|
|
default: |
|
dev_warn(chan2dev(edmac), "unknown interrupt!\n"); |
|
ret = IRQ_NONE; |
|
break; |
|
} |
|
|
|
spin_unlock(&edmac->lock); |
|
return ret; |
|
} |
|
|
|
/** |
|
* ep93xx_dma_tx_submit - set the prepared descriptor(s) to be executed |
|
* @tx: descriptor to be executed |
|
* |
|
* Function will execute given descriptor on the hardware or if the hardware |
|
* is busy, queue the descriptor to be executed later on. Returns cookie which |
|
* can be used to poll the status of the descriptor. |
|
*/ |
|
static dma_cookie_t ep93xx_dma_tx_submit(struct dma_async_tx_descriptor *tx) |
|
{ |
|
struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(tx->chan); |
|
struct ep93xx_dma_desc *desc; |
|
dma_cookie_t cookie; |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&edmac->lock, flags); |
|
cookie = dma_cookie_assign(tx); |
|
|
|
desc = container_of(tx, struct ep93xx_dma_desc, txd); |
|
|
|
/* |
|
* If nothing is currently prosessed, we push this descriptor |
|
* directly to the hardware. Otherwise we put the descriptor |
|
* to the pending queue. |
|
*/ |
|
if (list_empty(&edmac->active)) { |
|
ep93xx_dma_set_active(edmac, desc); |
|
edmac->edma->hw_submit(edmac); |
|
} else { |
|
list_add_tail(&desc->node, &edmac->queue); |
|
} |
|
|
|
spin_unlock_irqrestore(&edmac->lock, flags); |
|
return cookie; |
|
} |
|
|
|
/** |
|
* ep93xx_dma_alloc_chan_resources - allocate resources for the channel |
|
* @chan: channel to allocate resources |
|
* |
|
* Function allocates necessary resources for the given DMA channel and |
|
* returns number of allocated descriptors for the channel. Negative errno |
|
* is returned in case of failure. |
|
*/ |
|
static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan) |
|
{ |
|
struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan); |
|
struct ep93xx_dma_data *data = chan->private; |
|
const char *name = dma_chan_name(chan); |
|
int ret, i; |
|
|
|
/* Sanity check the channel parameters */ |
|
if (!edmac->edma->m2m) { |
|
if (!data) |
|
return -EINVAL; |
|
if (data->port < EP93XX_DMA_I2S1 || |
|
data->port > EP93XX_DMA_IRDA) |
|
return -EINVAL; |
|
if (data->direction != ep93xx_dma_chan_direction(chan)) |
|
return -EINVAL; |
|
} else { |
|
if (data) { |
|
switch (data->port) { |
|
case EP93XX_DMA_SSP: |
|
case EP93XX_DMA_IDE: |
|
if (!is_slave_direction(data->direction)) |
|
return -EINVAL; |
|
break; |
|
default: |
|
return -EINVAL; |
|
} |
|
} |
|
} |
|
|
|
if (data && data->name) |
|
name = data->name; |
|
|
|
ret = clk_enable(edmac->clk); |
|
if (ret) |
|
return ret; |
|
|
|
ret = request_irq(edmac->irq, ep93xx_dma_interrupt, 0, name, edmac); |
|
if (ret) |
|
goto fail_clk_disable; |
|
|
|
spin_lock_irq(&edmac->lock); |
|
dma_cookie_init(&edmac->chan); |
|
ret = edmac->edma->hw_setup(edmac); |
|
spin_unlock_irq(&edmac->lock); |
|
|
|
if (ret) |
|
goto fail_free_irq; |
|
|
|
for (i = 0; i < DMA_MAX_CHAN_DESCRIPTORS; i++) { |
|
struct ep93xx_dma_desc *desc; |
|
|
|
desc = kzalloc(sizeof(*desc), GFP_KERNEL); |
|
if (!desc) { |
|
dev_warn(chan2dev(edmac), "not enough descriptors\n"); |
|
break; |
|
} |
|
|
|
INIT_LIST_HEAD(&desc->tx_list); |
|
|
|
dma_async_tx_descriptor_init(&desc->txd, chan); |
|
desc->txd.flags = DMA_CTRL_ACK; |
|
desc->txd.tx_submit = ep93xx_dma_tx_submit; |
|
|
|
ep93xx_dma_desc_put(edmac, desc); |
|
} |
|
|
|
return i; |
|
|
|
fail_free_irq: |
|
free_irq(edmac->irq, edmac); |
|
fail_clk_disable: |
|
clk_disable(edmac->clk); |
|
|
|
return ret; |
|
} |
|
|
|
/** |
|
* ep93xx_dma_free_chan_resources - release resources for the channel |
|
* @chan: channel |
|
* |
|
* Function releases all the resources allocated for the given channel. |
|
* The channel must be idle when this is called. |
|
*/ |
|
static void ep93xx_dma_free_chan_resources(struct dma_chan *chan) |
|
{ |
|
struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan); |
|
struct ep93xx_dma_desc *desc, *d; |
|
unsigned long flags; |
|
LIST_HEAD(list); |
|
|
|
BUG_ON(!list_empty(&edmac->active)); |
|
BUG_ON(!list_empty(&edmac->queue)); |
|
|
|
spin_lock_irqsave(&edmac->lock, flags); |
|
edmac->edma->hw_shutdown(edmac); |
|
edmac->runtime_addr = 0; |
|
edmac->runtime_ctrl = 0; |
|
edmac->buffer = 0; |
|
list_splice_init(&edmac->free_list, &list); |
|
spin_unlock_irqrestore(&edmac->lock, flags); |
|
|
|
list_for_each_entry_safe(desc, d, &list, node) |
|
kfree(desc); |
|
|
|
clk_disable(edmac->clk); |
|
free_irq(edmac->irq, edmac); |
|
} |
|
|
|
/** |
|
* ep93xx_dma_prep_dma_memcpy - prepare a memcpy DMA operation |
|
* @chan: channel |
|
* @dest: destination bus address |
|
* @src: source bus address |
|
* @len: size of the transaction |
|
* @flags: flags for the descriptor |
|
* |
|
* Returns a valid DMA descriptor or %NULL in case of failure. |
|
*/ |
|
static struct dma_async_tx_descriptor * |
|
ep93xx_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, |
|
dma_addr_t src, size_t len, unsigned long flags) |
|
{ |
|
struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan); |
|
struct ep93xx_dma_desc *desc, *first; |
|
size_t bytes, offset; |
|
|
|
first = NULL; |
|
for (offset = 0; offset < len; offset += bytes) { |
|
desc = ep93xx_dma_desc_get(edmac); |
|
if (!desc) { |
|
dev_warn(chan2dev(edmac), "couldn't get descriptor\n"); |
|
goto fail; |
|
} |
|
|
|
bytes = min_t(size_t, len - offset, DMA_MAX_CHAN_BYTES); |
|
|
|
desc->src_addr = src + offset; |
|
desc->dst_addr = dest + offset; |
|
desc->size = bytes; |
|
|
|
if (!first) |
|
first = desc; |
|
else |
|
list_add_tail(&desc->node, &first->tx_list); |
|
} |
|
|
|
first->txd.cookie = -EBUSY; |
|
first->txd.flags = flags; |
|
|
|
return &first->txd; |
|
fail: |
|
ep93xx_dma_desc_put(edmac, first); |
|
return NULL; |
|
} |
|
|
|
/** |
|
* ep93xx_dma_prep_slave_sg - prepare a slave DMA operation |
|
* @chan: channel |
|
* @sgl: list of buffers to transfer |
|
* @sg_len: number of entries in @sgl |
|
* @dir: direction of tha DMA transfer |
|
* @flags: flags for the descriptor |
|
* @context: operation context (ignored) |
|
* |
|
* Returns a valid DMA descriptor or %NULL in case of failure. |
|
*/ |
|
static struct dma_async_tx_descriptor * |
|
ep93xx_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, |
|
unsigned int sg_len, enum dma_transfer_direction dir, |
|
unsigned long flags, void *context) |
|
{ |
|
struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan); |
|
struct ep93xx_dma_desc *desc, *first; |
|
struct scatterlist *sg; |
|
int i; |
|
|
|
if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) { |
|
dev_warn(chan2dev(edmac), |
|
"channel was configured with different direction\n"); |
|
return NULL; |
|
} |
|
|
|
if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) { |
|
dev_warn(chan2dev(edmac), |
|
"channel is already used for cyclic transfers\n"); |
|
return NULL; |
|
} |
|
|
|
ep93xx_dma_slave_config_write(chan, dir, &edmac->slave_config); |
|
|
|
first = NULL; |
|
for_each_sg(sgl, sg, sg_len, i) { |
|
size_t len = sg_dma_len(sg); |
|
|
|
if (len > DMA_MAX_CHAN_BYTES) { |
|
dev_warn(chan2dev(edmac), "too big transfer size %zu\n", |
|
len); |
|
goto fail; |
|
} |
|
|
|
desc = ep93xx_dma_desc_get(edmac); |
|
if (!desc) { |
|
dev_warn(chan2dev(edmac), "couldn't get descriptor\n"); |
|
goto fail; |
|
} |
|
|
|
if (dir == DMA_MEM_TO_DEV) { |
|
desc->src_addr = sg_dma_address(sg); |
|
desc->dst_addr = edmac->runtime_addr; |
|
} else { |
|
desc->src_addr = edmac->runtime_addr; |
|
desc->dst_addr = sg_dma_address(sg); |
|
} |
|
desc->size = len; |
|
|
|
if (!first) |
|
first = desc; |
|
else |
|
list_add_tail(&desc->node, &first->tx_list); |
|
} |
|
|
|
first->txd.cookie = -EBUSY; |
|
first->txd.flags = flags; |
|
|
|
return &first->txd; |
|
|
|
fail: |
|
ep93xx_dma_desc_put(edmac, first); |
|
return NULL; |
|
} |
|
|
|
/** |
|
* ep93xx_dma_prep_dma_cyclic - prepare a cyclic DMA operation |
|
* @chan: channel |
|
* @dma_addr: DMA mapped address of the buffer |
|
* @buf_len: length of the buffer (in bytes) |
|
* @period_len: length of a single period |
|
* @dir: direction of the operation |
|
* @flags: tx descriptor status flags |
|
* |
|
* Prepares a descriptor for cyclic DMA operation. This means that once the |
|
* descriptor is submitted, we will be submitting in a @period_len sized |
|
* buffers and calling callback once the period has been elapsed. Transfer |
|
* terminates only when client calls dmaengine_terminate_all() for this |
|
* channel. |
|
* |
|
* Returns a valid DMA descriptor or %NULL in case of failure. |
|
*/ |
|
static struct dma_async_tx_descriptor * |
|
ep93xx_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr, |
|
size_t buf_len, size_t period_len, |
|
enum dma_transfer_direction dir, unsigned long flags) |
|
{ |
|
struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan); |
|
struct ep93xx_dma_desc *desc, *first; |
|
size_t offset = 0; |
|
|
|
if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) { |
|
dev_warn(chan2dev(edmac), |
|
"channel was configured with different direction\n"); |
|
return NULL; |
|
} |
|
|
|
if (test_and_set_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) { |
|
dev_warn(chan2dev(edmac), |
|
"channel is already used for cyclic transfers\n"); |
|
return NULL; |
|
} |
|
|
|
if (period_len > DMA_MAX_CHAN_BYTES) { |
|
dev_warn(chan2dev(edmac), "too big period length %zu\n", |
|
period_len); |
|
return NULL; |
|
} |
|
|
|
ep93xx_dma_slave_config_write(chan, dir, &edmac->slave_config); |
|
|
|
/* Split the buffer into period size chunks */ |
|
first = NULL; |
|
for (offset = 0; offset < buf_len; offset += period_len) { |
|
desc = ep93xx_dma_desc_get(edmac); |
|
if (!desc) { |
|
dev_warn(chan2dev(edmac), "couldn't get descriptor\n"); |
|
goto fail; |
|
} |
|
|
|
if (dir == DMA_MEM_TO_DEV) { |
|
desc->src_addr = dma_addr + offset; |
|
desc->dst_addr = edmac->runtime_addr; |
|
} else { |
|
desc->src_addr = edmac->runtime_addr; |
|
desc->dst_addr = dma_addr + offset; |
|
} |
|
|
|
desc->size = period_len; |
|
|
|
if (!first) |
|
first = desc; |
|
else |
|
list_add_tail(&desc->node, &first->tx_list); |
|
} |
|
|
|
first->txd.cookie = -EBUSY; |
|
|
|
return &first->txd; |
|
|
|
fail: |
|
ep93xx_dma_desc_put(edmac, first); |
|
return NULL; |
|
} |
|
|
|
/** |
|
* ep93xx_dma_synchronize - Synchronizes the termination of transfers to the |
|
* current context. |
|
* @chan: channel |
|
* |
|
* Synchronizes the DMA channel termination to the current context. When this |
|
* function returns it is guaranteed that all transfers for previously issued |
|
* descriptors have stopped and and it is safe to free the memory associated |
|
* with them. Furthermore it is guaranteed that all complete callback functions |
|
* for a previously submitted descriptor have finished running and it is safe to |
|
* free resources accessed from within the complete callbacks. |
|
*/ |
|
static void ep93xx_dma_synchronize(struct dma_chan *chan) |
|
{ |
|
struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan); |
|
|
|
if (edmac->edma->hw_synchronize) |
|
edmac->edma->hw_synchronize(edmac); |
|
} |
|
|
|
/** |
|
* ep93xx_dma_terminate_all - terminate all transactions |
|
* @chan: channel |
|
* |
|
* Stops all DMA transactions. All descriptors are put back to the |
|
* @edmac->free_list and callbacks are _not_ called. |
|
*/ |
|
static int ep93xx_dma_terminate_all(struct dma_chan *chan) |
|
{ |
|
struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan); |
|
struct ep93xx_dma_desc *desc, *_d; |
|
unsigned long flags; |
|
LIST_HEAD(list); |
|
|
|
spin_lock_irqsave(&edmac->lock, flags); |
|
/* First we disable and flush the DMA channel */ |
|
edmac->edma->hw_shutdown(edmac); |
|
clear_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags); |
|
list_splice_init(&edmac->active, &list); |
|
list_splice_init(&edmac->queue, &list); |
|
/* |
|
* We then re-enable the channel. This way we can continue submitting |
|
* the descriptors by just calling ->hw_submit() again. |
|
*/ |
|
edmac->edma->hw_setup(edmac); |
|
spin_unlock_irqrestore(&edmac->lock, flags); |
|
|
|
list_for_each_entry_safe(desc, _d, &list, node) |
|
ep93xx_dma_desc_put(edmac, desc); |
|
|
|
return 0; |
|
} |
|
|
|
static int ep93xx_dma_slave_config(struct dma_chan *chan, |
|
struct dma_slave_config *config) |
|
{ |
|
struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan); |
|
|
|
memcpy(&edmac->slave_config, config, sizeof(*config)); |
|
|
|
return 0; |
|
} |
|
|
|
static int ep93xx_dma_slave_config_write(struct dma_chan *chan, |
|
enum dma_transfer_direction dir, |
|
struct dma_slave_config *config) |
|
{ |
|
struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan); |
|
enum dma_slave_buswidth width; |
|
unsigned long flags; |
|
u32 addr, ctrl; |
|
|
|
if (!edmac->edma->m2m) |
|
return -EINVAL; |
|
|
|
switch (dir) { |
|
case DMA_DEV_TO_MEM: |
|
width = config->src_addr_width; |
|
addr = config->src_addr; |
|
break; |
|
|
|
case DMA_MEM_TO_DEV: |
|
width = config->dst_addr_width; |
|
addr = config->dst_addr; |
|
break; |
|
|
|
default: |
|
return -EINVAL; |
|
} |
|
|
|
switch (width) { |
|
case DMA_SLAVE_BUSWIDTH_1_BYTE: |
|
ctrl = 0; |
|
break; |
|
case DMA_SLAVE_BUSWIDTH_2_BYTES: |
|
ctrl = M2M_CONTROL_PW_16; |
|
break; |
|
case DMA_SLAVE_BUSWIDTH_4_BYTES: |
|
ctrl = M2M_CONTROL_PW_32; |
|
break; |
|
default: |
|
return -EINVAL; |
|
} |
|
|
|
spin_lock_irqsave(&edmac->lock, flags); |
|
edmac->runtime_addr = addr; |
|
edmac->runtime_ctrl = ctrl; |
|
spin_unlock_irqrestore(&edmac->lock, flags); |
|
|
|
return 0; |
|
} |
|
|
|
/** |
|
* ep93xx_dma_tx_status - check if a transaction is completed |
|
* @chan: channel |
|
* @cookie: transaction specific cookie |
|
* @state: state of the transaction is stored here if given |
|
* |
|
* This function can be used to query state of a given transaction. |
|
*/ |
|
static enum dma_status ep93xx_dma_tx_status(struct dma_chan *chan, |
|
dma_cookie_t cookie, |
|
struct dma_tx_state *state) |
|
{ |
|
return dma_cookie_status(chan, cookie, state); |
|
} |
|
|
|
/** |
|
* ep93xx_dma_issue_pending - push pending transactions to the hardware |
|
* @chan: channel |
|
* |
|
* When this function is called, all pending transactions are pushed to the |
|
* hardware and executed. |
|
*/ |
|
static void ep93xx_dma_issue_pending(struct dma_chan *chan) |
|
{ |
|
ep93xx_dma_advance_work(to_ep93xx_dma_chan(chan)); |
|
} |
|
|
|
static int __init ep93xx_dma_probe(struct platform_device *pdev) |
|
{ |
|
struct ep93xx_dma_platform_data *pdata = dev_get_platdata(&pdev->dev); |
|
struct ep93xx_dma_engine *edma; |
|
struct dma_device *dma_dev; |
|
size_t edma_size; |
|
int ret, i; |
|
|
|
edma_size = pdata->num_channels * sizeof(struct ep93xx_dma_chan); |
|
edma = kzalloc(sizeof(*edma) + edma_size, GFP_KERNEL); |
|
if (!edma) |
|
return -ENOMEM; |
|
|
|
dma_dev = &edma->dma_dev; |
|
edma->m2m = platform_get_device_id(pdev)->driver_data; |
|
edma->num_channels = pdata->num_channels; |
|
|
|
INIT_LIST_HEAD(&dma_dev->channels); |
|
for (i = 0; i < pdata->num_channels; i++) { |
|
const struct ep93xx_dma_chan_data *cdata = &pdata->channels[i]; |
|
struct ep93xx_dma_chan *edmac = &edma->channels[i]; |
|
|
|
edmac->chan.device = dma_dev; |
|
edmac->regs = cdata->base; |
|
edmac->irq = cdata->irq; |
|
edmac->edma = edma; |
|
|
|
edmac->clk = clk_get(NULL, cdata->name); |
|
if (IS_ERR(edmac->clk)) { |
|
dev_warn(&pdev->dev, "failed to get clock for %s\n", |
|
cdata->name); |
|
continue; |
|
} |
|
|
|
spin_lock_init(&edmac->lock); |
|
INIT_LIST_HEAD(&edmac->active); |
|
INIT_LIST_HEAD(&edmac->queue); |
|
INIT_LIST_HEAD(&edmac->free_list); |
|
tasklet_setup(&edmac->tasklet, ep93xx_dma_tasklet); |
|
|
|
list_add_tail(&edmac->chan.device_node, |
|
&dma_dev->channels); |
|
} |
|
|
|
dma_cap_zero(dma_dev->cap_mask); |
|
dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); |
|
dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); |
|
|
|
dma_dev->dev = &pdev->dev; |
|
dma_dev->device_alloc_chan_resources = ep93xx_dma_alloc_chan_resources; |
|
dma_dev->device_free_chan_resources = ep93xx_dma_free_chan_resources; |
|
dma_dev->device_prep_slave_sg = ep93xx_dma_prep_slave_sg; |
|
dma_dev->device_prep_dma_cyclic = ep93xx_dma_prep_dma_cyclic; |
|
dma_dev->device_config = ep93xx_dma_slave_config; |
|
dma_dev->device_synchronize = ep93xx_dma_synchronize; |
|
dma_dev->device_terminate_all = ep93xx_dma_terminate_all; |
|
dma_dev->device_issue_pending = ep93xx_dma_issue_pending; |
|
dma_dev->device_tx_status = ep93xx_dma_tx_status; |
|
|
|
dma_set_max_seg_size(dma_dev->dev, DMA_MAX_CHAN_BYTES); |
|
|
|
if (edma->m2m) { |
|
dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); |
|
dma_dev->device_prep_dma_memcpy = ep93xx_dma_prep_dma_memcpy; |
|
|
|
edma->hw_setup = m2m_hw_setup; |
|
edma->hw_shutdown = m2m_hw_shutdown; |
|
edma->hw_submit = m2m_hw_submit; |
|
edma->hw_interrupt = m2m_hw_interrupt; |
|
} else { |
|
dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); |
|
|
|
edma->hw_synchronize = m2p_hw_synchronize; |
|
edma->hw_setup = m2p_hw_setup; |
|
edma->hw_shutdown = m2p_hw_shutdown; |
|
edma->hw_submit = m2p_hw_submit; |
|
edma->hw_interrupt = m2p_hw_interrupt; |
|
} |
|
|
|
ret = dma_async_device_register(dma_dev); |
|
if (unlikely(ret)) { |
|
for (i = 0; i < edma->num_channels; i++) { |
|
struct ep93xx_dma_chan *edmac = &edma->channels[i]; |
|
if (!IS_ERR_OR_NULL(edmac->clk)) |
|
clk_put(edmac->clk); |
|
} |
|
kfree(edma); |
|
} else { |
|
dev_info(dma_dev->dev, "EP93xx M2%s DMA ready\n", |
|
edma->m2m ? "M" : "P"); |
|
} |
|
|
|
return ret; |
|
} |
|
|
|
static const struct platform_device_id ep93xx_dma_driver_ids[] = { |
|
{ "ep93xx-dma-m2p", 0 }, |
|
{ "ep93xx-dma-m2m", 1 }, |
|
{ }, |
|
}; |
|
|
|
static struct platform_driver ep93xx_dma_driver = { |
|
.driver = { |
|
.name = "ep93xx-dma", |
|
}, |
|
.id_table = ep93xx_dma_driver_ids, |
|
}; |
|
|
|
static int __init ep93xx_dma_module_init(void) |
|
{ |
|
return platform_driver_probe(&ep93xx_dma_driver, ep93xx_dma_probe); |
|
} |
|
subsys_initcall(ep93xx_dma_module_init); |
|
|
|
MODULE_AUTHOR("Mika Westerberg <[email protected]>"); |
|
MODULE_DESCRIPTION("EP93xx DMA driver"); |
|
MODULE_LICENSE("GPL");
|
|
|