forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1278 lines
32 KiB
1278 lines
32 KiB
// SPDX-License-Identifier: GPL-2.0+ |
|
// |
|
// Actions Semi Owl SoCs DMA driver |
|
// |
|
// Copyright (c) 2014 Actions Semi Inc. |
|
// Author: David Liu <[email protected]> |
|
// |
|
// Copyright (c) 2018 Linaro Ltd. |
|
// Author: Manivannan Sadhasivam <[email protected]> |
|
|
|
#include <linux/bitops.h> |
|
#include <linux/clk.h> |
|
#include <linux/delay.h> |
|
#include <linux/dmaengine.h> |
|
#include <linux/dma-mapping.h> |
|
#include <linux/dmapool.h> |
|
#include <linux/err.h> |
|
#include <linux/init.h> |
|
#include <linux/interrupt.h> |
|
#include <linux/io.h> |
|
#include <linux/mm.h> |
|
#include <linux/module.h> |
|
#include <linux/of_device.h> |
|
#include <linux/of_dma.h> |
|
#include <linux/slab.h> |
|
#include "virt-dma.h" |
|
|
|
#define OWL_DMA_FRAME_MAX_LENGTH 0xfffff |
|
|
|
/* Global DMA Controller Registers */ |
|
#define OWL_DMA_IRQ_PD0 0x00 |
|
#define OWL_DMA_IRQ_PD1 0x04 |
|
#define OWL_DMA_IRQ_PD2 0x08 |
|
#define OWL_DMA_IRQ_PD3 0x0C |
|
#define OWL_DMA_IRQ_EN0 0x10 |
|
#define OWL_DMA_IRQ_EN1 0x14 |
|
#define OWL_DMA_IRQ_EN2 0x18 |
|
#define OWL_DMA_IRQ_EN3 0x1C |
|
#define OWL_DMA_SECURE_ACCESS_CTL 0x20 |
|
#define OWL_DMA_NIC_QOS 0x24 |
|
#define OWL_DMA_DBGSEL 0x28 |
|
#define OWL_DMA_IDLE_STAT 0x2C |
|
|
|
/* Channel Registers */ |
|
#define OWL_DMA_CHAN_BASE(i) (0x100 + (i) * 0x100) |
|
#define OWL_DMAX_MODE 0x00 |
|
#define OWL_DMAX_SOURCE 0x04 |
|
#define OWL_DMAX_DESTINATION 0x08 |
|
#define OWL_DMAX_FRAME_LEN 0x0C |
|
#define OWL_DMAX_FRAME_CNT 0x10 |
|
#define OWL_DMAX_REMAIN_FRAME_CNT 0x14 |
|
#define OWL_DMAX_REMAIN_CNT 0x18 |
|
#define OWL_DMAX_SOURCE_STRIDE 0x1C |
|
#define OWL_DMAX_DESTINATION_STRIDE 0x20 |
|
#define OWL_DMAX_START 0x24 |
|
#define OWL_DMAX_PAUSE 0x28 |
|
#define OWL_DMAX_CHAINED_CTL 0x2C |
|
#define OWL_DMAX_CONSTANT 0x30 |
|
#define OWL_DMAX_LINKLIST_CTL 0x34 |
|
#define OWL_DMAX_NEXT_DESCRIPTOR 0x38 |
|
#define OWL_DMAX_CURRENT_DESCRIPTOR_NUM 0x3C |
|
#define OWL_DMAX_INT_CTL 0x40 |
|
#define OWL_DMAX_INT_STATUS 0x44 |
|
#define OWL_DMAX_CURRENT_SOURCE_POINTER 0x48 |
|
#define OWL_DMAX_CURRENT_DESTINATION_POINTER 0x4C |
|
|
|
/* OWL_DMAX_MODE Bits */ |
|
#define OWL_DMA_MODE_TS(x) (((x) & GENMASK(5, 0)) << 0) |
|
#define OWL_DMA_MODE_ST(x) (((x) & GENMASK(1, 0)) << 8) |
|
#define OWL_DMA_MODE_ST_DEV OWL_DMA_MODE_ST(0) |
|
#define OWL_DMA_MODE_ST_DCU OWL_DMA_MODE_ST(2) |
|
#define OWL_DMA_MODE_ST_SRAM OWL_DMA_MODE_ST(3) |
|
#define OWL_DMA_MODE_DT(x) (((x) & GENMASK(1, 0)) << 10) |
|
#define OWL_DMA_MODE_DT_DEV OWL_DMA_MODE_DT(0) |
|
#define OWL_DMA_MODE_DT_DCU OWL_DMA_MODE_DT(2) |
|
#define OWL_DMA_MODE_DT_SRAM OWL_DMA_MODE_DT(3) |
|
#define OWL_DMA_MODE_SAM(x) (((x) & GENMASK(1, 0)) << 16) |
|
#define OWL_DMA_MODE_SAM_CONST OWL_DMA_MODE_SAM(0) |
|
#define OWL_DMA_MODE_SAM_INC OWL_DMA_MODE_SAM(1) |
|
#define OWL_DMA_MODE_SAM_STRIDE OWL_DMA_MODE_SAM(2) |
|
#define OWL_DMA_MODE_DAM(x) (((x) & GENMASK(1, 0)) << 18) |
|
#define OWL_DMA_MODE_DAM_CONST OWL_DMA_MODE_DAM(0) |
|
#define OWL_DMA_MODE_DAM_INC OWL_DMA_MODE_DAM(1) |
|
#define OWL_DMA_MODE_DAM_STRIDE OWL_DMA_MODE_DAM(2) |
|
#define OWL_DMA_MODE_PW(x) (((x) & GENMASK(2, 0)) << 20) |
|
#define OWL_DMA_MODE_CB BIT(23) |
|
#define OWL_DMA_MODE_NDDBW(x) (((x) & 0x1) << 28) |
|
#define OWL_DMA_MODE_NDDBW_32BIT OWL_DMA_MODE_NDDBW(0) |
|
#define OWL_DMA_MODE_NDDBW_8BIT OWL_DMA_MODE_NDDBW(1) |
|
#define OWL_DMA_MODE_CFE BIT(29) |
|
#define OWL_DMA_MODE_LME BIT(30) |
|
#define OWL_DMA_MODE_CME BIT(31) |
|
|
|
/* OWL_DMAX_LINKLIST_CTL Bits */ |
|
#define OWL_DMA_LLC_SAV(x) (((x) & GENMASK(1, 0)) << 8) |
|
#define OWL_DMA_LLC_SAV_INC OWL_DMA_LLC_SAV(0) |
|
#define OWL_DMA_LLC_SAV_LOAD_NEXT OWL_DMA_LLC_SAV(1) |
|
#define OWL_DMA_LLC_SAV_LOAD_PREV OWL_DMA_LLC_SAV(2) |
|
#define OWL_DMA_LLC_DAV(x) (((x) & GENMASK(1, 0)) << 10) |
|
#define OWL_DMA_LLC_DAV_INC OWL_DMA_LLC_DAV(0) |
|
#define OWL_DMA_LLC_DAV_LOAD_NEXT OWL_DMA_LLC_DAV(1) |
|
#define OWL_DMA_LLC_DAV_LOAD_PREV OWL_DMA_LLC_DAV(2) |
|
#define OWL_DMA_LLC_SUSPEND BIT(16) |
|
|
|
/* OWL_DMAX_INT_CTL Bits */ |
|
#define OWL_DMA_INTCTL_BLOCK BIT(0) |
|
#define OWL_DMA_INTCTL_SUPER_BLOCK BIT(1) |
|
#define OWL_DMA_INTCTL_FRAME BIT(2) |
|
#define OWL_DMA_INTCTL_HALF_FRAME BIT(3) |
|
#define OWL_DMA_INTCTL_LAST_FRAME BIT(4) |
|
|
|
/* OWL_DMAX_INT_STATUS Bits */ |
|
#define OWL_DMA_INTSTAT_BLOCK BIT(0) |
|
#define OWL_DMA_INTSTAT_SUPER_BLOCK BIT(1) |
|
#define OWL_DMA_INTSTAT_FRAME BIT(2) |
|
#define OWL_DMA_INTSTAT_HALF_FRAME BIT(3) |
|
#define OWL_DMA_INTSTAT_LAST_FRAME BIT(4) |
|
|
|
/* Pack shift and newshift in a single word */ |
|
#define BIT_FIELD(val, width, shift, newshift) \ |
|
((((val) >> (shift)) & ((BIT(width)) - 1)) << (newshift)) |
|
|
|
/* Frame count value is fixed as 1 */ |
|
#define FCNT_VAL 0x1 |
|
|
|
/** |
|
* enum owl_dmadesc_offsets - Describe DMA descriptor, hardware link |
|
* list for dma transfer |
|
* @OWL_DMADESC_NEXT_LLI: physical address of the next link list |
|
* @OWL_DMADESC_SADDR: source physical address |
|
* @OWL_DMADESC_DADDR: destination physical address |
|
* @OWL_DMADESC_FLEN: frame length |
|
* @OWL_DMADESC_SRC_STRIDE: source stride |
|
* @OWL_DMADESC_DST_STRIDE: destination stride |
|
* @OWL_DMADESC_CTRLA: dma_mode and linklist ctrl config |
|
* @OWL_DMADESC_CTRLB: interrupt config |
|
* @OWL_DMADESC_CONST_NUM: data for constant fill |
|
* @OWL_DMADESC_SIZE: max size of this enum |
|
*/ |
|
enum owl_dmadesc_offsets { |
|
OWL_DMADESC_NEXT_LLI = 0, |
|
OWL_DMADESC_SADDR, |
|
OWL_DMADESC_DADDR, |
|
OWL_DMADESC_FLEN, |
|
OWL_DMADESC_SRC_STRIDE, |
|
OWL_DMADESC_DST_STRIDE, |
|
OWL_DMADESC_CTRLA, |
|
OWL_DMADESC_CTRLB, |
|
OWL_DMADESC_CONST_NUM, |
|
OWL_DMADESC_SIZE |
|
}; |
|
|
|
enum owl_dma_id { |
|
S900_DMA, |
|
S700_DMA, |
|
}; |
|
|
|
/** |
|
* struct owl_dma_lli - Link list for dma transfer |
|
* @hw: hardware link list |
|
* @phys: physical address of hardware link list |
|
* @node: node for txd's lli_list |
|
*/ |
|
struct owl_dma_lli { |
|
u32 hw[OWL_DMADESC_SIZE]; |
|
dma_addr_t phys; |
|
struct list_head node; |
|
}; |
|
|
|
/** |
|
* struct owl_dma_txd - Wrapper for struct dma_async_tx_descriptor |
|
* @vd: virtual DMA descriptor |
|
* @lli_list: link list of lli nodes |
|
* @cyclic: flag to indicate cyclic transfers |
|
*/ |
|
struct owl_dma_txd { |
|
struct virt_dma_desc vd; |
|
struct list_head lli_list; |
|
bool cyclic; |
|
}; |
|
|
|
/** |
|
* struct owl_dma_pchan - Holder for the physical channels |
|
* @id: physical index to this channel |
|
* @base: virtual memory base for the dma channel |
|
* @vchan: the virtual channel currently being served by this physical channel |
|
*/ |
|
struct owl_dma_pchan { |
|
u32 id; |
|
void __iomem *base; |
|
struct owl_dma_vchan *vchan; |
|
}; |
|
|
|
/** |
|
* struct owl_dma_pchan - Wrapper for DMA ENGINE channel |
|
* @vc: wrappped virtual channel |
|
* @pchan: the physical channel utilized by this channel |
|
* @txd: active transaction on this channel |
|
* @cfg: slave configuration for this channel |
|
* @drq: physical DMA request ID for this channel |
|
*/ |
|
struct owl_dma_vchan { |
|
struct virt_dma_chan vc; |
|
struct owl_dma_pchan *pchan; |
|
struct owl_dma_txd *txd; |
|
struct dma_slave_config cfg; |
|
u8 drq; |
|
}; |
|
|
|
/** |
|
* struct owl_dma - Holder for the Owl DMA controller |
|
* @dma: dma engine for this instance |
|
* @base: virtual memory base for the DMA controller |
|
* @clk: clock for the DMA controller |
|
* @lock: a lock to use when change DMA controller global register |
|
* @lli_pool: a pool for the LLI descriptors |
|
* @irq: interrupt ID for the DMA controller |
|
* @nr_pchans: the number of physical channels |
|
* @pchans: array of data for the physical channels |
|
* @nr_vchans: the number of physical channels |
|
* @vchans: array of data for the physical channels |
|
* @devid: device id based on OWL SoC |
|
*/ |
|
struct owl_dma { |
|
struct dma_device dma; |
|
void __iomem *base; |
|
struct clk *clk; |
|
spinlock_t lock; |
|
struct dma_pool *lli_pool; |
|
int irq; |
|
|
|
unsigned int nr_pchans; |
|
struct owl_dma_pchan *pchans; |
|
|
|
unsigned int nr_vchans; |
|
struct owl_dma_vchan *vchans; |
|
enum owl_dma_id devid; |
|
}; |
|
|
|
static void pchan_update(struct owl_dma_pchan *pchan, u32 reg, |
|
u32 val, bool state) |
|
{ |
|
u32 regval; |
|
|
|
regval = readl(pchan->base + reg); |
|
|
|
if (state) |
|
regval |= val; |
|
else |
|
regval &= ~val; |
|
|
|
writel(val, pchan->base + reg); |
|
} |
|
|
|
static void pchan_writel(struct owl_dma_pchan *pchan, u32 reg, u32 data) |
|
{ |
|
writel(data, pchan->base + reg); |
|
} |
|
|
|
static u32 pchan_readl(struct owl_dma_pchan *pchan, u32 reg) |
|
{ |
|
return readl(pchan->base + reg); |
|
} |
|
|
|
static void dma_update(struct owl_dma *od, u32 reg, u32 val, bool state) |
|
{ |
|
u32 regval; |
|
|
|
regval = readl(od->base + reg); |
|
|
|
if (state) |
|
regval |= val; |
|
else |
|
regval &= ~val; |
|
|
|
writel(val, od->base + reg); |
|
} |
|
|
|
static void dma_writel(struct owl_dma *od, u32 reg, u32 data) |
|
{ |
|
writel(data, od->base + reg); |
|
} |
|
|
|
static u32 dma_readl(struct owl_dma *od, u32 reg) |
|
{ |
|
return readl(od->base + reg); |
|
} |
|
|
|
static inline struct owl_dma *to_owl_dma(struct dma_device *dd) |
|
{ |
|
return container_of(dd, struct owl_dma, dma); |
|
} |
|
|
|
static struct device *chan2dev(struct dma_chan *chan) |
|
{ |
|
return &chan->dev->device; |
|
} |
|
|
|
static inline struct owl_dma_vchan *to_owl_vchan(struct dma_chan *chan) |
|
{ |
|
return container_of(chan, struct owl_dma_vchan, vc.chan); |
|
} |
|
|
|
static inline struct owl_dma_txd *to_owl_txd(struct dma_async_tx_descriptor *tx) |
|
{ |
|
return container_of(tx, struct owl_dma_txd, vd.tx); |
|
} |
|
|
|
static inline u32 llc_hw_ctrla(u32 mode, u32 llc_ctl) |
|
{ |
|
u32 ctl; |
|
|
|
ctl = BIT_FIELD(mode, 4, 28, 28) | |
|
BIT_FIELD(mode, 8, 16, 20) | |
|
BIT_FIELD(mode, 4, 8, 16) | |
|
BIT_FIELD(mode, 6, 0, 10) | |
|
BIT_FIELD(llc_ctl, 2, 10, 8) | |
|
BIT_FIELD(llc_ctl, 2, 8, 6); |
|
|
|
return ctl; |
|
} |
|
|
|
static inline u32 llc_hw_ctrlb(u32 int_ctl) |
|
{ |
|
u32 ctl; |
|
|
|
/* |
|
* Irrespective of the SoC, ctrlb value starts filling from |
|
* bit 18. |
|
*/ |
|
ctl = BIT_FIELD(int_ctl, 7, 0, 18); |
|
|
|
return ctl; |
|
} |
|
|
|
static u32 llc_hw_flen(struct owl_dma_lli *lli) |
|
{ |
|
return lli->hw[OWL_DMADESC_FLEN] & GENMASK(19, 0); |
|
} |
|
|
|
static void owl_dma_free_lli(struct owl_dma *od, |
|
struct owl_dma_lli *lli) |
|
{ |
|
list_del(&lli->node); |
|
dma_pool_free(od->lli_pool, lli, lli->phys); |
|
} |
|
|
|
static struct owl_dma_lli *owl_dma_alloc_lli(struct owl_dma *od) |
|
{ |
|
struct owl_dma_lli *lli; |
|
dma_addr_t phys; |
|
|
|
lli = dma_pool_alloc(od->lli_pool, GFP_NOWAIT, &phys); |
|
if (!lli) |
|
return NULL; |
|
|
|
INIT_LIST_HEAD(&lli->node); |
|
lli->phys = phys; |
|
|
|
return lli; |
|
} |
|
|
|
static struct owl_dma_lli *owl_dma_add_lli(struct owl_dma_txd *txd, |
|
struct owl_dma_lli *prev, |
|
struct owl_dma_lli *next, |
|
bool is_cyclic) |
|
{ |
|
if (!is_cyclic) |
|
list_add_tail(&next->node, &txd->lli_list); |
|
|
|
if (prev) { |
|
prev->hw[OWL_DMADESC_NEXT_LLI] = next->phys; |
|
prev->hw[OWL_DMADESC_CTRLA] |= |
|
llc_hw_ctrla(OWL_DMA_MODE_LME, 0); |
|
} |
|
|
|
return next; |
|
} |
|
|
|
static inline int owl_dma_cfg_lli(struct owl_dma_vchan *vchan, |
|
struct owl_dma_lli *lli, |
|
dma_addr_t src, dma_addr_t dst, |
|
u32 len, enum dma_transfer_direction dir, |
|
struct dma_slave_config *sconfig, |
|
bool is_cyclic) |
|
{ |
|
struct owl_dma *od = to_owl_dma(vchan->vc.chan.device); |
|
u32 mode, ctrlb; |
|
|
|
mode = OWL_DMA_MODE_PW(0); |
|
|
|
switch (dir) { |
|
case DMA_MEM_TO_MEM: |
|
mode |= OWL_DMA_MODE_TS(0) | OWL_DMA_MODE_ST_DCU | |
|
OWL_DMA_MODE_DT_DCU | OWL_DMA_MODE_SAM_INC | |
|
OWL_DMA_MODE_DAM_INC; |
|
|
|
break; |
|
case DMA_MEM_TO_DEV: |
|
mode |= OWL_DMA_MODE_TS(vchan->drq) |
|
| OWL_DMA_MODE_ST_DCU | OWL_DMA_MODE_DT_DEV |
|
| OWL_DMA_MODE_SAM_INC | OWL_DMA_MODE_DAM_CONST; |
|
|
|
/* |
|
* Hardware only supports 32bit and 8bit buswidth. Since the |
|
* default is 32bit, select 8bit only when requested. |
|
*/ |
|
if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE) |
|
mode |= OWL_DMA_MODE_NDDBW_8BIT; |
|
|
|
break; |
|
case DMA_DEV_TO_MEM: |
|
mode |= OWL_DMA_MODE_TS(vchan->drq) |
|
| OWL_DMA_MODE_ST_DEV | OWL_DMA_MODE_DT_DCU |
|
| OWL_DMA_MODE_SAM_CONST | OWL_DMA_MODE_DAM_INC; |
|
|
|
/* |
|
* Hardware only supports 32bit and 8bit buswidth. Since the |
|
* default is 32bit, select 8bit only when requested. |
|
*/ |
|
if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE) |
|
mode |= OWL_DMA_MODE_NDDBW_8BIT; |
|
|
|
break; |
|
default: |
|
return -EINVAL; |
|
} |
|
|
|
lli->hw[OWL_DMADESC_CTRLA] = llc_hw_ctrla(mode, |
|
OWL_DMA_LLC_SAV_LOAD_NEXT | |
|
OWL_DMA_LLC_DAV_LOAD_NEXT); |
|
|
|
if (is_cyclic) |
|
ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_BLOCK); |
|
else |
|
ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_SUPER_BLOCK); |
|
|
|
lli->hw[OWL_DMADESC_NEXT_LLI] = 0; /* One link list by default */ |
|
lli->hw[OWL_DMADESC_SADDR] = src; |
|
lli->hw[OWL_DMADESC_DADDR] = dst; |
|
lli->hw[OWL_DMADESC_SRC_STRIDE] = 0; |
|
lli->hw[OWL_DMADESC_DST_STRIDE] = 0; |
|
|
|
if (od->devid == S700_DMA) { |
|
/* Max frame length is 1MB */ |
|
lli->hw[OWL_DMADESC_FLEN] = len; |
|
/* |
|
* On S700, word starts from offset 0x1C is shared between |
|
* frame count and ctrlb, where first 12 bits are for frame |
|
* count and rest of 20 bits are for ctrlb. |
|
*/ |
|
lli->hw[OWL_DMADESC_CTRLB] = FCNT_VAL | ctrlb; |
|
} else { |
|
/* |
|
* On S900, word starts from offset 0xC is shared between |
|
* frame length (max frame length is 1MB) and frame count, |
|
* where first 20 bits are for frame length and rest of |
|
* 12 bits are for frame count. |
|
*/ |
|
lli->hw[OWL_DMADESC_FLEN] = len | FCNT_VAL << 20; |
|
lli->hw[OWL_DMADESC_CTRLB] = ctrlb; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
static struct owl_dma_pchan *owl_dma_get_pchan(struct owl_dma *od, |
|
struct owl_dma_vchan *vchan) |
|
{ |
|
struct owl_dma_pchan *pchan = NULL; |
|
unsigned long flags; |
|
int i; |
|
|
|
for (i = 0; i < od->nr_pchans; i++) { |
|
pchan = &od->pchans[i]; |
|
|
|
spin_lock_irqsave(&od->lock, flags); |
|
if (!pchan->vchan) { |
|
pchan->vchan = vchan; |
|
spin_unlock_irqrestore(&od->lock, flags); |
|
break; |
|
} |
|
|
|
spin_unlock_irqrestore(&od->lock, flags); |
|
} |
|
|
|
return pchan; |
|
} |
|
|
|
static int owl_dma_pchan_busy(struct owl_dma *od, struct owl_dma_pchan *pchan) |
|
{ |
|
unsigned int val; |
|
|
|
val = dma_readl(od, OWL_DMA_IDLE_STAT); |
|
|
|
return !(val & (1 << pchan->id)); |
|
} |
|
|
|
static void owl_dma_terminate_pchan(struct owl_dma *od, |
|
struct owl_dma_pchan *pchan) |
|
{ |
|
unsigned long flags; |
|
u32 irq_pd; |
|
|
|
pchan_writel(pchan, OWL_DMAX_START, 0); |
|
pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false); |
|
|
|
spin_lock_irqsave(&od->lock, flags); |
|
dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), false); |
|
|
|
irq_pd = dma_readl(od, OWL_DMA_IRQ_PD0); |
|
if (irq_pd & (1 << pchan->id)) { |
|
dev_warn(od->dma.dev, |
|
"terminating pchan %d that still has pending irq\n", |
|
pchan->id); |
|
dma_writel(od, OWL_DMA_IRQ_PD0, (1 << pchan->id)); |
|
} |
|
|
|
pchan->vchan = NULL; |
|
|
|
spin_unlock_irqrestore(&od->lock, flags); |
|
} |
|
|
|
static void owl_dma_pause_pchan(struct owl_dma_pchan *pchan) |
|
{ |
|
pchan_writel(pchan, 1, OWL_DMAX_PAUSE); |
|
} |
|
|
|
static void owl_dma_resume_pchan(struct owl_dma_pchan *pchan) |
|
{ |
|
pchan_writel(pchan, 0, OWL_DMAX_PAUSE); |
|
} |
|
|
|
static int owl_dma_start_next_txd(struct owl_dma_vchan *vchan) |
|
{ |
|
struct owl_dma *od = to_owl_dma(vchan->vc.chan.device); |
|
struct virt_dma_desc *vd = vchan_next_desc(&vchan->vc); |
|
struct owl_dma_pchan *pchan = vchan->pchan; |
|
struct owl_dma_txd *txd = to_owl_txd(&vd->tx); |
|
struct owl_dma_lli *lli; |
|
unsigned long flags; |
|
u32 int_ctl; |
|
|
|
list_del(&vd->node); |
|
|
|
vchan->txd = txd; |
|
|
|
/* Wait for channel inactive */ |
|
while (owl_dma_pchan_busy(od, pchan)) |
|
cpu_relax(); |
|
|
|
lli = list_first_entry(&txd->lli_list, |
|
struct owl_dma_lli, node); |
|
|
|
if (txd->cyclic) |
|
int_ctl = OWL_DMA_INTCTL_BLOCK; |
|
else |
|
int_ctl = OWL_DMA_INTCTL_SUPER_BLOCK; |
|
|
|
pchan_writel(pchan, OWL_DMAX_MODE, OWL_DMA_MODE_LME); |
|
pchan_writel(pchan, OWL_DMAX_LINKLIST_CTL, |
|
OWL_DMA_LLC_SAV_LOAD_NEXT | OWL_DMA_LLC_DAV_LOAD_NEXT); |
|
pchan_writel(pchan, OWL_DMAX_NEXT_DESCRIPTOR, lli->phys); |
|
pchan_writel(pchan, OWL_DMAX_INT_CTL, int_ctl); |
|
|
|
/* Clear IRQ status for this pchan */ |
|
pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false); |
|
|
|
spin_lock_irqsave(&od->lock, flags); |
|
|
|
dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), true); |
|
|
|
spin_unlock_irqrestore(&od->lock, flags); |
|
|
|
dev_dbg(chan2dev(&vchan->vc.chan), "starting pchan %d\n", pchan->id); |
|
|
|
/* Start DMA transfer for this pchan */ |
|
pchan_writel(pchan, OWL_DMAX_START, 0x1); |
|
|
|
return 0; |
|
} |
|
|
|
static void owl_dma_phy_free(struct owl_dma *od, struct owl_dma_vchan *vchan) |
|
{ |
|
/* Ensure that the physical channel is stopped */ |
|
owl_dma_terminate_pchan(od, vchan->pchan); |
|
|
|
vchan->pchan = NULL; |
|
} |
|
|
|
static irqreturn_t owl_dma_interrupt(int irq, void *dev_id) |
|
{ |
|
struct owl_dma *od = dev_id; |
|
struct owl_dma_vchan *vchan; |
|
struct owl_dma_pchan *pchan; |
|
unsigned long pending; |
|
int i; |
|
unsigned int global_irq_pending, chan_irq_pending; |
|
|
|
spin_lock(&od->lock); |
|
|
|
pending = dma_readl(od, OWL_DMA_IRQ_PD0); |
|
|
|
/* Clear IRQ status for each pchan */ |
|
for_each_set_bit(i, &pending, od->nr_pchans) { |
|
pchan = &od->pchans[i]; |
|
pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false); |
|
} |
|
|
|
/* Clear pending IRQ */ |
|
dma_writel(od, OWL_DMA_IRQ_PD0, pending); |
|
|
|
/* Check missed pending IRQ */ |
|
for (i = 0; i < od->nr_pchans; i++) { |
|
pchan = &od->pchans[i]; |
|
chan_irq_pending = pchan_readl(pchan, OWL_DMAX_INT_CTL) & |
|
pchan_readl(pchan, OWL_DMAX_INT_STATUS); |
|
|
|
/* Dummy read to ensure OWL_DMA_IRQ_PD0 value is updated */ |
|
dma_readl(od, OWL_DMA_IRQ_PD0); |
|
|
|
global_irq_pending = dma_readl(od, OWL_DMA_IRQ_PD0); |
|
|
|
if (chan_irq_pending && !(global_irq_pending & BIT(i))) { |
|
dev_dbg(od->dma.dev, |
|
"global and channel IRQ pending match err\n"); |
|
|
|
/* Clear IRQ status for this pchan */ |
|
pchan_update(pchan, OWL_DMAX_INT_STATUS, |
|
0xff, false); |
|
|
|
/* Update global IRQ pending */ |
|
pending |= BIT(i); |
|
} |
|
} |
|
|
|
spin_unlock(&od->lock); |
|
|
|
for_each_set_bit(i, &pending, od->nr_pchans) { |
|
struct owl_dma_txd *txd; |
|
|
|
pchan = &od->pchans[i]; |
|
|
|
vchan = pchan->vchan; |
|
if (!vchan) { |
|
dev_warn(od->dma.dev, "no vchan attached on pchan %d\n", |
|
pchan->id); |
|
continue; |
|
} |
|
|
|
spin_lock(&vchan->vc.lock); |
|
|
|
txd = vchan->txd; |
|
if (txd) { |
|
vchan->txd = NULL; |
|
|
|
vchan_cookie_complete(&txd->vd); |
|
|
|
/* |
|
* Start the next descriptor (if any), |
|
* otherwise free this channel. |
|
*/ |
|
if (vchan_next_desc(&vchan->vc)) |
|
owl_dma_start_next_txd(vchan); |
|
else |
|
owl_dma_phy_free(od, vchan); |
|
} |
|
|
|
spin_unlock(&vchan->vc.lock); |
|
} |
|
|
|
return IRQ_HANDLED; |
|
} |
|
|
|
static void owl_dma_free_txd(struct owl_dma *od, struct owl_dma_txd *txd) |
|
{ |
|
struct owl_dma_lli *lli, *_lli; |
|
|
|
if (unlikely(!txd)) |
|
return; |
|
|
|
list_for_each_entry_safe(lli, _lli, &txd->lli_list, node) |
|
owl_dma_free_lli(od, lli); |
|
|
|
kfree(txd); |
|
} |
|
|
|
static void owl_dma_desc_free(struct virt_dma_desc *vd) |
|
{ |
|
struct owl_dma *od = to_owl_dma(vd->tx.chan->device); |
|
struct owl_dma_txd *txd = to_owl_txd(&vd->tx); |
|
|
|
owl_dma_free_txd(od, txd); |
|
} |
|
|
|
static int owl_dma_terminate_all(struct dma_chan *chan) |
|
{ |
|
struct owl_dma *od = to_owl_dma(chan->device); |
|
struct owl_dma_vchan *vchan = to_owl_vchan(chan); |
|
unsigned long flags; |
|
LIST_HEAD(head); |
|
|
|
spin_lock_irqsave(&vchan->vc.lock, flags); |
|
|
|
if (vchan->pchan) |
|
owl_dma_phy_free(od, vchan); |
|
|
|
if (vchan->txd) { |
|
owl_dma_desc_free(&vchan->txd->vd); |
|
vchan->txd = NULL; |
|
} |
|
|
|
vchan_get_all_descriptors(&vchan->vc, &head); |
|
|
|
spin_unlock_irqrestore(&vchan->vc.lock, flags); |
|
|
|
vchan_dma_desc_free_list(&vchan->vc, &head); |
|
|
|
return 0; |
|
} |
|
|
|
static int owl_dma_config(struct dma_chan *chan, |
|
struct dma_slave_config *config) |
|
{ |
|
struct owl_dma_vchan *vchan = to_owl_vchan(chan); |
|
|
|
/* Reject definitely invalid configurations */ |
|
if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || |
|
config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) |
|
return -EINVAL; |
|
|
|
memcpy(&vchan->cfg, config, sizeof(struct dma_slave_config)); |
|
|
|
return 0; |
|
} |
|
|
|
static int owl_dma_pause(struct dma_chan *chan) |
|
{ |
|
struct owl_dma_vchan *vchan = to_owl_vchan(chan); |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&vchan->vc.lock, flags); |
|
|
|
owl_dma_pause_pchan(vchan->pchan); |
|
|
|
spin_unlock_irqrestore(&vchan->vc.lock, flags); |
|
|
|
return 0; |
|
} |
|
|
|
static int owl_dma_resume(struct dma_chan *chan) |
|
{ |
|
struct owl_dma_vchan *vchan = to_owl_vchan(chan); |
|
unsigned long flags; |
|
|
|
if (!vchan->pchan && !vchan->txd) |
|
return 0; |
|
|
|
dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc); |
|
|
|
spin_lock_irqsave(&vchan->vc.lock, flags); |
|
|
|
owl_dma_resume_pchan(vchan->pchan); |
|
|
|
spin_unlock_irqrestore(&vchan->vc.lock, flags); |
|
|
|
return 0; |
|
} |
|
|
|
static u32 owl_dma_getbytes_chan(struct owl_dma_vchan *vchan) |
|
{ |
|
struct owl_dma_pchan *pchan; |
|
struct owl_dma_txd *txd; |
|
struct owl_dma_lli *lli; |
|
unsigned int next_lli_phy; |
|
size_t bytes; |
|
|
|
pchan = vchan->pchan; |
|
txd = vchan->txd; |
|
|
|
if (!pchan || !txd) |
|
return 0; |
|
|
|
/* Get remain count of current node in link list */ |
|
bytes = pchan_readl(pchan, OWL_DMAX_REMAIN_CNT); |
|
|
|
/* Loop through the preceding nodes to get total remaining bytes */ |
|
if (pchan_readl(pchan, OWL_DMAX_MODE) & OWL_DMA_MODE_LME) { |
|
next_lli_phy = pchan_readl(pchan, OWL_DMAX_NEXT_DESCRIPTOR); |
|
list_for_each_entry(lli, &txd->lli_list, node) { |
|
/* Start from the next active node */ |
|
if (lli->phys == next_lli_phy) { |
|
list_for_each_entry(lli, &txd->lli_list, node) |
|
bytes += llc_hw_flen(lli); |
|
break; |
|
} |
|
} |
|
} |
|
|
|
return bytes; |
|
} |
|
|
|
static enum dma_status owl_dma_tx_status(struct dma_chan *chan, |
|
dma_cookie_t cookie, |
|
struct dma_tx_state *state) |
|
{ |
|
struct owl_dma_vchan *vchan = to_owl_vchan(chan); |
|
struct owl_dma_lli *lli; |
|
struct virt_dma_desc *vd; |
|
struct owl_dma_txd *txd; |
|
enum dma_status ret; |
|
unsigned long flags; |
|
size_t bytes = 0; |
|
|
|
ret = dma_cookie_status(chan, cookie, state); |
|
if (ret == DMA_COMPLETE || !state) |
|
return ret; |
|
|
|
spin_lock_irqsave(&vchan->vc.lock, flags); |
|
|
|
vd = vchan_find_desc(&vchan->vc, cookie); |
|
if (vd) { |
|
txd = to_owl_txd(&vd->tx); |
|
list_for_each_entry(lli, &txd->lli_list, node) |
|
bytes += llc_hw_flen(lli); |
|
} else { |
|
bytes = owl_dma_getbytes_chan(vchan); |
|
} |
|
|
|
spin_unlock_irqrestore(&vchan->vc.lock, flags); |
|
|
|
dma_set_residue(state, bytes); |
|
|
|
return ret; |
|
} |
|
|
|
static void owl_dma_phy_alloc_and_start(struct owl_dma_vchan *vchan) |
|
{ |
|
struct owl_dma *od = to_owl_dma(vchan->vc.chan.device); |
|
struct owl_dma_pchan *pchan; |
|
|
|
pchan = owl_dma_get_pchan(od, vchan); |
|
if (!pchan) |
|
return; |
|
|
|
dev_dbg(od->dma.dev, "allocated pchan %d\n", pchan->id); |
|
|
|
vchan->pchan = pchan; |
|
owl_dma_start_next_txd(vchan); |
|
} |
|
|
|
static void owl_dma_issue_pending(struct dma_chan *chan) |
|
{ |
|
struct owl_dma_vchan *vchan = to_owl_vchan(chan); |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&vchan->vc.lock, flags); |
|
if (vchan_issue_pending(&vchan->vc)) { |
|
if (!vchan->pchan) |
|
owl_dma_phy_alloc_and_start(vchan); |
|
} |
|
spin_unlock_irqrestore(&vchan->vc.lock, flags); |
|
} |
|
|
|
static struct dma_async_tx_descriptor |
|
*owl_dma_prep_memcpy(struct dma_chan *chan, |
|
dma_addr_t dst, dma_addr_t src, |
|
size_t len, unsigned long flags) |
|
{ |
|
struct owl_dma *od = to_owl_dma(chan->device); |
|
struct owl_dma_vchan *vchan = to_owl_vchan(chan); |
|
struct owl_dma_txd *txd; |
|
struct owl_dma_lli *lli, *prev = NULL; |
|
size_t offset, bytes; |
|
int ret; |
|
|
|
if (!len) |
|
return NULL; |
|
|
|
txd = kzalloc(sizeof(*txd), GFP_NOWAIT); |
|
if (!txd) |
|
return NULL; |
|
|
|
INIT_LIST_HEAD(&txd->lli_list); |
|
|
|
/* Process the transfer as frame by frame */ |
|
for (offset = 0; offset < len; offset += bytes) { |
|
lli = owl_dma_alloc_lli(od); |
|
if (!lli) { |
|
dev_warn(chan2dev(chan), "failed to allocate lli\n"); |
|
goto err_txd_free; |
|
} |
|
|
|
bytes = min_t(size_t, (len - offset), OWL_DMA_FRAME_MAX_LENGTH); |
|
|
|
ret = owl_dma_cfg_lli(vchan, lli, src + offset, dst + offset, |
|
bytes, DMA_MEM_TO_MEM, |
|
&vchan->cfg, txd->cyclic); |
|
if (ret) { |
|
dev_warn(chan2dev(chan), "failed to config lli\n"); |
|
goto err_txd_free; |
|
} |
|
|
|
prev = owl_dma_add_lli(txd, prev, lli, false); |
|
} |
|
|
|
return vchan_tx_prep(&vchan->vc, &txd->vd, flags); |
|
|
|
err_txd_free: |
|
owl_dma_free_txd(od, txd); |
|
return NULL; |
|
} |
|
|
|
static struct dma_async_tx_descriptor |
|
*owl_dma_prep_slave_sg(struct dma_chan *chan, |
|
struct scatterlist *sgl, |
|
unsigned int sg_len, |
|
enum dma_transfer_direction dir, |
|
unsigned long flags, void *context) |
|
{ |
|
struct owl_dma *od = to_owl_dma(chan->device); |
|
struct owl_dma_vchan *vchan = to_owl_vchan(chan); |
|
struct dma_slave_config *sconfig = &vchan->cfg; |
|
struct owl_dma_txd *txd; |
|
struct owl_dma_lli *lli, *prev = NULL; |
|
struct scatterlist *sg; |
|
dma_addr_t addr, src = 0, dst = 0; |
|
size_t len; |
|
int ret, i; |
|
|
|
txd = kzalloc(sizeof(*txd), GFP_NOWAIT); |
|
if (!txd) |
|
return NULL; |
|
|
|
INIT_LIST_HEAD(&txd->lli_list); |
|
|
|
for_each_sg(sgl, sg, sg_len, i) { |
|
addr = sg_dma_address(sg); |
|
len = sg_dma_len(sg); |
|
|
|
if (len > OWL_DMA_FRAME_MAX_LENGTH) { |
|
dev_err(od->dma.dev, |
|
"frame length exceeds max supported length"); |
|
goto err_txd_free; |
|
} |
|
|
|
lli = owl_dma_alloc_lli(od); |
|
if (!lli) { |
|
dev_err(chan2dev(chan), "failed to allocate lli"); |
|
goto err_txd_free; |
|
} |
|
|
|
if (dir == DMA_MEM_TO_DEV) { |
|
src = addr; |
|
dst = sconfig->dst_addr; |
|
} else { |
|
src = sconfig->src_addr; |
|
dst = addr; |
|
} |
|
|
|
ret = owl_dma_cfg_lli(vchan, lli, src, dst, len, dir, sconfig, |
|
txd->cyclic); |
|
if (ret) { |
|
dev_warn(chan2dev(chan), "failed to config lli"); |
|
goto err_txd_free; |
|
} |
|
|
|
prev = owl_dma_add_lli(txd, prev, lli, false); |
|
} |
|
|
|
return vchan_tx_prep(&vchan->vc, &txd->vd, flags); |
|
|
|
err_txd_free: |
|
owl_dma_free_txd(od, txd); |
|
|
|
return NULL; |
|
} |
|
|
|
static struct dma_async_tx_descriptor |
|
*owl_prep_dma_cyclic(struct dma_chan *chan, |
|
dma_addr_t buf_addr, size_t buf_len, |
|
size_t period_len, |
|
enum dma_transfer_direction dir, |
|
unsigned long flags) |
|
{ |
|
struct owl_dma *od = to_owl_dma(chan->device); |
|
struct owl_dma_vchan *vchan = to_owl_vchan(chan); |
|
struct dma_slave_config *sconfig = &vchan->cfg; |
|
struct owl_dma_txd *txd; |
|
struct owl_dma_lli *lli, *prev = NULL, *first = NULL; |
|
dma_addr_t src = 0, dst = 0; |
|
unsigned int periods = buf_len / period_len; |
|
int ret, i; |
|
|
|
txd = kzalloc(sizeof(*txd), GFP_NOWAIT); |
|
if (!txd) |
|
return NULL; |
|
|
|
INIT_LIST_HEAD(&txd->lli_list); |
|
txd->cyclic = true; |
|
|
|
for (i = 0; i < periods; i++) { |
|
lli = owl_dma_alloc_lli(od); |
|
if (!lli) { |
|
dev_warn(chan2dev(chan), "failed to allocate lli"); |
|
goto err_txd_free; |
|
} |
|
|
|
if (dir == DMA_MEM_TO_DEV) { |
|
src = buf_addr + (period_len * i); |
|
dst = sconfig->dst_addr; |
|
} else if (dir == DMA_DEV_TO_MEM) { |
|
src = sconfig->src_addr; |
|
dst = buf_addr + (period_len * i); |
|
} |
|
|
|
ret = owl_dma_cfg_lli(vchan, lli, src, dst, period_len, |
|
dir, sconfig, txd->cyclic); |
|
if (ret) { |
|
dev_warn(chan2dev(chan), "failed to config lli"); |
|
goto err_txd_free; |
|
} |
|
|
|
if (!first) |
|
first = lli; |
|
|
|
prev = owl_dma_add_lli(txd, prev, lli, false); |
|
} |
|
|
|
/* close the cyclic list */ |
|
owl_dma_add_lli(txd, prev, first, true); |
|
|
|
return vchan_tx_prep(&vchan->vc, &txd->vd, flags); |
|
|
|
err_txd_free: |
|
owl_dma_free_txd(od, txd); |
|
|
|
return NULL; |
|
} |
|
|
|
static void owl_dma_free_chan_resources(struct dma_chan *chan) |
|
{ |
|
struct owl_dma_vchan *vchan = to_owl_vchan(chan); |
|
|
|
/* Ensure all queued descriptors are freed */ |
|
vchan_free_chan_resources(&vchan->vc); |
|
} |
|
|
|
static inline void owl_dma_free(struct owl_dma *od) |
|
{ |
|
struct owl_dma_vchan *vchan = NULL; |
|
struct owl_dma_vchan *next; |
|
|
|
list_for_each_entry_safe(vchan, |
|
next, &od->dma.channels, vc.chan.device_node) { |
|
list_del(&vchan->vc.chan.device_node); |
|
tasklet_kill(&vchan->vc.task); |
|
} |
|
} |
|
|
|
static struct dma_chan *owl_dma_of_xlate(struct of_phandle_args *dma_spec, |
|
struct of_dma *ofdma) |
|
{ |
|
struct owl_dma *od = ofdma->of_dma_data; |
|
struct owl_dma_vchan *vchan; |
|
struct dma_chan *chan; |
|
u8 drq = dma_spec->args[0]; |
|
|
|
if (drq > od->nr_vchans) |
|
return NULL; |
|
|
|
chan = dma_get_any_slave_channel(&od->dma); |
|
if (!chan) |
|
return NULL; |
|
|
|
vchan = to_owl_vchan(chan); |
|
vchan->drq = drq; |
|
|
|
return chan; |
|
} |
|
|
|
static const struct of_device_id owl_dma_match[] = { |
|
{ .compatible = "actions,s500-dma", .data = (void *)S900_DMA,}, |
|
{ .compatible = "actions,s700-dma", .data = (void *)S700_DMA,}, |
|
{ .compatible = "actions,s900-dma", .data = (void *)S900_DMA,}, |
|
{ /* sentinel */ }, |
|
}; |
|
MODULE_DEVICE_TABLE(of, owl_dma_match); |
|
|
|
static int owl_dma_probe(struct platform_device *pdev) |
|
{ |
|
struct device_node *np = pdev->dev.of_node; |
|
struct owl_dma *od; |
|
int ret, i, nr_channels, nr_requests; |
|
|
|
od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); |
|
if (!od) |
|
return -ENOMEM; |
|
|
|
od->base = devm_platform_ioremap_resource(pdev, 0); |
|
if (IS_ERR(od->base)) |
|
return PTR_ERR(od->base); |
|
|
|
ret = of_property_read_u32(np, "dma-channels", &nr_channels); |
|
if (ret) { |
|
dev_err(&pdev->dev, "can't get dma-channels\n"); |
|
return ret; |
|
} |
|
|
|
ret = of_property_read_u32(np, "dma-requests", &nr_requests); |
|
if (ret) { |
|
dev_err(&pdev->dev, "can't get dma-requests\n"); |
|
return ret; |
|
} |
|
|
|
dev_info(&pdev->dev, "dma-channels %d, dma-requests %d\n", |
|
nr_channels, nr_requests); |
|
|
|
od->devid = (enum owl_dma_id)of_device_get_match_data(&pdev->dev); |
|
|
|
od->nr_pchans = nr_channels; |
|
od->nr_vchans = nr_requests; |
|
|
|
pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32); |
|
|
|
platform_set_drvdata(pdev, od); |
|
spin_lock_init(&od->lock); |
|
|
|
dma_cap_set(DMA_MEMCPY, od->dma.cap_mask); |
|
dma_cap_set(DMA_SLAVE, od->dma.cap_mask); |
|
dma_cap_set(DMA_CYCLIC, od->dma.cap_mask); |
|
|
|
od->dma.dev = &pdev->dev; |
|
od->dma.device_free_chan_resources = owl_dma_free_chan_resources; |
|
od->dma.device_tx_status = owl_dma_tx_status; |
|
od->dma.device_issue_pending = owl_dma_issue_pending; |
|
od->dma.device_prep_dma_memcpy = owl_dma_prep_memcpy; |
|
od->dma.device_prep_slave_sg = owl_dma_prep_slave_sg; |
|
od->dma.device_prep_dma_cyclic = owl_prep_dma_cyclic; |
|
od->dma.device_config = owl_dma_config; |
|
od->dma.device_pause = owl_dma_pause; |
|
od->dma.device_resume = owl_dma_resume; |
|
od->dma.device_terminate_all = owl_dma_terminate_all; |
|
od->dma.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
|
od->dma.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
|
od->dma.directions = BIT(DMA_MEM_TO_MEM); |
|
od->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; |
|
|
|
INIT_LIST_HEAD(&od->dma.channels); |
|
|
|
od->clk = devm_clk_get(&pdev->dev, NULL); |
|
if (IS_ERR(od->clk)) { |
|
dev_err(&pdev->dev, "unable to get clock\n"); |
|
return PTR_ERR(od->clk); |
|
} |
|
|
|
/* |
|
* Eventhough the DMA controller is capable of generating 4 |
|
* IRQ's for DMA priority feature, we only use 1 IRQ for |
|
* simplification. |
|
*/ |
|
od->irq = platform_get_irq(pdev, 0); |
|
ret = devm_request_irq(&pdev->dev, od->irq, owl_dma_interrupt, 0, |
|
dev_name(&pdev->dev), od); |
|
if (ret) { |
|
dev_err(&pdev->dev, "unable to request IRQ\n"); |
|
return ret; |
|
} |
|
|
|
/* Init physical channel */ |
|
od->pchans = devm_kcalloc(&pdev->dev, od->nr_pchans, |
|
sizeof(struct owl_dma_pchan), GFP_KERNEL); |
|
if (!od->pchans) |
|
return -ENOMEM; |
|
|
|
for (i = 0; i < od->nr_pchans; i++) { |
|
struct owl_dma_pchan *pchan = &od->pchans[i]; |
|
|
|
pchan->id = i; |
|
pchan->base = od->base + OWL_DMA_CHAN_BASE(i); |
|
} |
|
|
|
/* Init virtual channel */ |
|
od->vchans = devm_kcalloc(&pdev->dev, od->nr_vchans, |
|
sizeof(struct owl_dma_vchan), GFP_KERNEL); |
|
if (!od->vchans) |
|
return -ENOMEM; |
|
|
|
for (i = 0; i < od->nr_vchans; i++) { |
|
struct owl_dma_vchan *vchan = &od->vchans[i]; |
|
|
|
vchan->vc.desc_free = owl_dma_desc_free; |
|
vchan_init(&vchan->vc, &od->dma); |
|
} |
|
|
|
/* Create a pool of consistent memory blocks for hardware descriptors */ |
|
od->lli_pool = dma_pool_create(dev_name(od->dma.dev), od->dma.dev, |
|
sizeof(struct owl_dma_lli), |
|
__alignof__(struct owl_dma_lli), |
|
0); |
|
if (!od->lli_pool) { |
|
dev_err(&pdev->dev, "unable to allocate DMA descriptor pool\n"); |
|
return -ENOMEM; |
|
} |
|
|
|
clk_prepare_enable(od->clk); |
|
|
|
ret = dma_async_device_register(&od->dma); |
|
if (ret) { |
|
dev_err(&pdev->dev, "failed to register DMA engine device\n"); |
|
goto err_pool_free; |
|
} |
|
|
|
/* Device-tree DMA controller registration */ |
|
ret = of_dma_controller_register(pdev->dev.of_node, |
|
owl_dma_of_xlate, od); |
|
if (ret) { |
|
dev_err(&pdev->dev, "of_dma_controller_register failed\n"); |
|
goto err_dma_unregister; |
|
} |
|
|
|
return 0; |
|
|
|
err_dma_unregister: |
|
dma_async_device_unregister(&od->dma); |
|
err_pool_free: |
|
clk_disable_unprepare(od->clk); |
|
dma_pool_destroy(od->lli_pool); |
|
|
|
return ret; |
|
} |
|
|
|
static int owl_dma_remove(struct platform_device *pdev) |
|
{ |
|
struct owl_dma *od = platform_get_drvdata(pdev); |
|
|
|
of_dma_controller_free(pdev->dev.of_node); |
|
dma_async_device_unregister(&od->dma); |
|
|
|
/* Mask all interrupts for this execution environment */ |
|
dma_writel(od, OWL_DMA_IRQ_EN0, 0x0); |
|
|
|
/* Make sure we won't have any further interrupts */ |
|
devm_free_irq(od->dma.dev, od->irq, od); |
|
|
|
owl_dma_free(od); |
|
|
|
clk_disable_unprepare(od->clk); |
|
dma_pool_destroy(od->lli_pool); |
|
|
|
return 0; |
|
} |
|
|
|
static struct platform_driver owl_dma_driver = { |
|
.probe = owl_dma_probe, |
|
.remove = owl_dma_remove, |
|
.driver = { |
|
.name = "dma-owl", |
|
.of_match_table = of_match_ptr(owl_dma_match), |
|
}, |
|
}; |
|
|
|
static int owl_dma_init(void) |
|
{ |
|
return platform_driver_register(&owl_dma_driver); |
|
} |
|
subsys_initcall(owl_dma_init); |
|
|
|
static void __exit owl_dma_exit(void) |
|
{ |
|
platform_driver_unregister(&owl_dma_driver); |
|
} |
|
module_exit(owl_dma_exit); |
|
|
|
MODULE_AUTHOR("David Liu <[email protected]>"); |
|
MODULE_AUTHOR("Manivannan Sadhasivam <[email protected]>"); |
|
MODULE_DESCRIPTION("Actions Semi Owl SoCs DMA driver"); |
|
MODULE_LICENSE("GPL");
|
|
|