mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
855 lines
23 KiB
855 lines
23 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* |
|
* Qualcomm Technologies HIDMA DMA engine low level code |
|
* |
|
* Copyright (c) 2015-2016, The Linux Foundation. All rights reserved. |
|
*/ |
|
|
|
#include <linux/dmaengine.h> |
|
#include <linux/slab.h> |
|
#include <linux/interrupt.h> |
|
#include <linux/mm.h> |
|
#include <linux/highmem.h> |
|
#include <linux/dma-mapping.h> |
|
#include <linux/delay.h> |
|
#include <linux/atomic.h> |
|
#include <linux/iopoll.h> |
|
#include <linux/kfifo.h> |
|
#include <linux/bitops.h> |
|
|
|
#include "hidma.h" |
|
|
|
#define HIDMA_EVRE_SIZE 16 /* each EVRE is 16 bytes */ |
|
|
|
#define HIDMA_TRCA_CTRLSTS_REG 0x000 |
|
#define HIDMA_TRCA_RING_LOW_REG 0x008 |
|
#define HIDMA_TRCA_RING_HIGH_REG 0x00C |
|
#define HIDMA_TRCA_RING_LEN_REG 0x010 |
|
#define HIDMA_TRCA_DOORBELL_REG 0x400 |
|
|
|
#define HIDMA_EVCA_CTRLSTS_REG 0x000 |
|
#define HIDMA_EVCA_INTCTRL_REG 0x004 |
|
#define HIDMA_EVCA_RING_LOW_REG 0x008 |
|
#define HIDMA_EVCA_RING_HIGH_REG 0x00C |
|
#define HIDMA_EVCA_RING_LEN_REG 0x010 |
|
#define HIDMA_EVCA_WRITE_PTR_REG 0x020 |
|
#define HIDMA_EVCA_DOORBELL_REG 0x400 |
|
|
|
#define HIDMA_EVCA_IRQ_STAT_REG 0x100 |
|
#define HIDMA_EVCA_IRQ_CLR_REG 0x108 |
|
#define HIDMA_EVCA_IRQ_EN_REG 0x110 |
|
|
|
#define HIDMA_EVRE_CFG_IDX 0 |
|
|
|
#define HIDMA_EVRE_ERRINFO_BIT_POS 24 |
|
#define HIDMA_EVRE_CODE_BIT_POS 28 |
|
|
|
#define HIDMA_EVRE_ERRINFO_MASK GENMASK(3, 0) |
|
#define HIDMA_EVRE_CODE_MASK GENMASK(3, 0) |
|
|
|
#define HIDMA_CH_CONTROL_MASK GENMASK(7, 0) |
|
#define HIDMA_CH_STATE_MASK GENMASK(7, 0) |
|
#define HIDMA_CH_STATE_BIT_POS 0x8 |
|
|
|
#define HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS 0 |
|
#define HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS 1 |
|
#define HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS 9 |
|
#define HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS 10 |
|
#define HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS 11 |
|
#define HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS 14 |
|
|
|
#define ENABLE_IRQS (BIT(HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS) | \ |
|
BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \ |
|
BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \ |
|
BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \ |
|
BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS) | \ |
|
BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS)) |
|
|
|
#define HIDMA_INCREMENT_ITERATOR(iter, size, ring_size) \ |
|
do { \ |
|
iter += size; \ |
|
if (iter >= ring_size) \ |
|
iter -= ring_size; \ |
|
} while (0) |
|
|
|
#define HIDMA_CH_STATE(val) \ |
|
((val >> HIDMA_CH_STATE_BIT_POS) & HIDMA_CH_STATE_MASK) |
|
|
|
#define HIDMA_ERR_INT_MASK \ |
|
(BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS) | \ |
|
BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \ |
|
BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \ |
|
BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \ |
|
BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS)) |
|
|
|
enum ch_command { |
|
HIDMA_CH_DISABLE = 0, |
|
HIDMA_CH_ENABLE = 1, |
|
HIDMA_CH_SUSPEND = 2, |
|
HIDMA_CH_RESET = 9, |
|
}; |
|
|
|
enum ch_state { |
|
HIDMA_CH_DISABLED = 0, |
|
HIDMA_CH_ENABLED = 1, |
|
HIDMA_CH_RUNNING = 2, |
|
HIDMA_CH_SUSPENDED = 3, |
|
HIDMA_CH_STOPPED = 4, |
|
}; |
|
|
|
enum err_code { |
|
HIDMA_EVRE_STATUS_COMPLETE = 1, |
|
HIDMA_EVRE_STATUS_ERROR = 4, |
|
}; |
|
|
|
static int hidma_is_chan_enabled(int state) |
|
{ |
|
switch (state) { |
|
case HIDMA_CH_ENABLED: |
|
case HIDMA_CH_RUNNING: |
|
return true; |
|
default: |
|
return false; |
|
} |
|
} |
|
|
|
void hidma_ll_free(struct hidma_lldev *lldev, u32 tre_ch) |
|
{ |
|
struct hidma_tre *tre; |
|
|
|
if (tre_ch >= lldev->nr_tres) { |
|
dev_err(lldev->dev, "invalid TRE number in free:%d", tre_ch); |
|
return; |
|
} |
|
|
|
tre = &lldev->trepool[tre_ch]; |
|
if (atomic_read(&tre->allocated) != true) { |
|
dev_err(lldev->dev, "trying to free an unused TRE:%d", tre_ch); |
|
return; |
|
} |
|
|
|
atomic_set(&tre->allocated, 0); |
|
} |
|
|
|
int hidma_ll_request(struct hidma_lldev *lldev, u32 sig, const char *dev_name, |
|
void (*callback)(void *data), void *data, u32 *tre_ch) |
|
{ |
|
unsigned int i; |
|
struct hidma_tre *tre; |
|
u32 *tre_local; |
|
|
|
if (!tre_ch || !lldev) |
|
return -EINVAL; |
|
|
|
/* need to have at least one empty spot in the queue */ |
|
for (i = 0; i < lldev->nr_tres - 1; i++) { |
|
if (atomic_add_unless(&lldev->trepool[i].allocated, 1, 1)) |
|
break; |
|
} |
|
|
|
if (i == (lldev->nr_tres - 1)) |
|
return -ENOMEM; |
|
|
|
tre = &lldev->trepool[i]; |
|
tre->dma_sig = sig; |
|
tre->dev_name = dev_name; |
|
tre->callback = callback; |
|
tre->data = data; |
|
tre->idx = i; |
|
tre->status = 0; |
|
tre->queued = 0; |
|
tre->err_code = 0; |
|
tre->err_info = 0; |
|
tre->lldev = lldev; |
|
tre_local = &tre->tre_local[0]; |
|
tre_local[HIDMA_TRE_CFG_IDX] = (lldev->chidx & 0xFF) << 8; |
|
tre_local[HIDMA_TRE_CFG_IDX] |= BIT(16); /* set IEOB */ |
|
*tre_ch = i; |
|
if (callback) |
|
callback(data); |
|
return 0; |
|
} |
|
|
|
/* |
|
* Multiple TREs may be queued and waiting in the pending queue. |
|
*/ |
|
static void hidma_ll_tre_complete(struct tasklet_struct *t) |
|
{ |
|
struct hidma_lldev *lldev = from_tasklet(lldev, t, task); |
|
struct hidma_tre *tre; |
|
|
|
while (kfifo_out(&lldev->handoff_fifo, &tre, 1)) { |
|
/* call the user if it has been read by the hardware */ |
|
if (tre->callback) |
|
tre->callback(tre->data); |
|
} |
|
} |
|
|
|
static int hidma_post_completed(struct hidma_lldev *lldev, u8 err_info, |
|
u8 err_code) |
|
{ |
|
struct hidma_tre *tre; |
|
unsigned long flags; |
|
u32 tre_iterator; |
|
|
|
spin_lock_irqsave(&lldev->lock, flags); |
|
|
|
tre_iterator = lldev->tre_processed_off; |
|
tre = lldev->pending_tre_list[tre_iterator / HIDMA_TRE_SIZE]; |
|
if (!tre) { |
|
spin_unlock_irqrestore(&lldev->lock, flags); |
|
dev_warn(lldev->dev, "tre_index [%d] and tre out of sync\n", |
|
tre_iterator / HIDMA_TRE_SIZE); |
|
return -EINVAL; |
|
} |
|
lldev->pending_tre_list[tre->tre_index] = NULL; |
|
|
|
/* |
|
* Keep track of pending TREs that SW is expecting to receive |
|
* from HW. We got one now. Decrement our counter. |
|
*/ |
|
if (atomic_dec_return(&lldev->pending_tre_count) < 0) { |
|
dev_warn(lldev->dev, "tre count mismatch on completion"); |
|
atomic_set(&lldev->pending_tre_count, 0); |
|
} |
|
|
|
HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE, |
|
lldev->tre_ring_size); |
|
lldev->tre_processed_off = tre_iterator; |
|
spin_unlock_irqrestore(&lldev->lock, flags); |
|
|
|
tre->err_info = err_info; |
|
tre->err_code = err_code; |
|
tre->queued = 0; |
|
|
|
kfifo_put(&lldev->handoff_fifo, tre); |
|
tasklet_schedule(&lldev->task); |
|
|
|
return 0; |
|
} |
|
|
|
/* |
|
* Called to handle the interrupt for the channel. |
|
* Return a positive number if TRE or EVRE were consumed on this run. |
|
* Return a positive number if there are pending TREs or EVREs. |
|
* Return 0 if there is nothing to consume or no pending TREs/EVREs found. |
|
*/ |
|
static int hidma_handle_tre_completion(struct hidma_lldev *lldev) |
|
{ |
|
u32 evre_ring_size = lldev->evre_ring_size; |
|
u32 err_info, err_code, evre_write_off; |
|
u32 evre_iterator; |
|
u32 num_completed = 0; |
|
|
|
evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG); |
|
evre_iterator = lldev->evre_processed_off; |
|
|
|
if ((evre_write_off > evre_ring_size) || |
|
(evre_write_off % HIDMA_EVRE_SIZE)) { |
|
dev_err(lldev->dev, "HW reports invalid EVRE write offset\n"); |
|
return 0; |
|
} |
|
|
|
/* |
|
* By the time control reaches here the number of EVREs and TREs |
|
* may not match. Only consume the ones that hardware told us. |
|
*/ |
|
while ((evre_iterator != evre_write_off)) { |
|
u32 *current_evre = lldev->evre_ring + evre_iterator; |
|
u32 cfg; |
|
|
|
cfg = current_evre[HIDMA_EVRE_CFG_IDX]; |
|
err_info = cfg >> HIDMA_EVRE_ERRINFO_BIT_POS; |
|
err_info &= HIDMA_EVRE_ERRINFO_MASK; |
|
err_code = |
|
(cfg >> HIDMA_EVRE_CODE_BIT_POS) & HIDMA_EVRE_CODE_MASK; |
|
|
|
if (hidma_post_completed(lldev, err_info, err_code)) |
|
break; |
|
|
|
HIDMA_INCREMENT_ITERATOR(evre_iterator, HIDMA_EVRE_SIZE, |
|
evre_ring_size); |
|
|
|
/* |
|
* Read the new event descriptor written by the HW. |
|
* As we are processing the delivered events, other events |
|
* get queued to the SW for processing. |
|
*/ |
|
evre_write_off = |
|
readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG); |
|
num_completed++; |
|
|
|
/* |
|
* An error interrupt might have arrived while we are processing |
|
* the completed interrupt. |
|
*/ |
|
if (!hidma_ll_isenabled(lldev)) |
|
break; |
|
} |
|
|
|
if (num_completed) { |
|
u32 evre_read_off = (lldev->evre_processed_off + |
|
HIDMA_EVRE_SIZE * num_completed); |
|
evre_read_off = evre_read_off % evre_ring_size; |
|
writel(evre_read_off, lldev->evca + HIDMA_EVCA_DOORBELL_REG); |
|
|
|
/* record the last processed tre offset */ |
|
lldev->evre_processed_off = evre_read_off; |
|
} |
|
|
|
return num_completed; |
|
} |
|
|
|
void hidma_cleanup_pending_tre(struct hidma_lldev *lldev, u8 err_info, |
|
u8 err_code) |
|
{ |
|
while (atomic_read(&lldev->pending_tre_count)) { |
|
if (hidma_post_completed(lldev, err_info, err_code)) |
|
break; |
|
} |
|
} |
|
|
|
static int hidma_ll_reset(struct hidma_lldev *lldev) |
|
{ |
|
u32 val; |
|
int ret; |
|
|
|
val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
|
val &= ~(HIDMA_CH_CONTROL_MASK << 16); |
|
val |= HIDMA_CH_RESET << 16; |
|
writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
|
|
|
/* |
|
* Delay 10ms after reset to allow DMA logic to quiesce. |
|
* Do a polled read up to 1ms and 10ms maximum. |
|
*/ |
|
ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, |
|
HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED, |
|
1000, 10000); |
|
if (ret) { |
|
dev_err(lldev->dev, "transfer channel did not reset\n"); |
|
return ret; |
|
} |
|
|
|
val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
|
val &= ~(HIDMA_CH_CONTROL_MASK << 16); |
|
val |= HIDMA_CH_RESET << 16; |
|
writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
|
|
|
/* |
|
* Delay 10ms after reset to allow DMA logic to quiesce. |
|
* Do a polled read up to 1ms and 10ms maximum. |
|
*/ |
|
ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, |
|
HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED, |
|
1000, 10000); |
|
if (ret) |
|
return ret; |
|
|
|
lldev->trch_state = HIDMA_CH_DISABLED; |
|
lldev->evch_state = HIDMA_CH_DISABLED; |
|
return 0; |
|
} |
|
|
|
/* |
|
* The interrupt handler for HIDMA will try to consume as many pending |
|
* EVRE from the event queue as possible. Each EVRE has an associated |
|
* TRE that holds the user interface parameters. EVRE reports the |
|
* result of the transaction. Hardware guarantees ordering between EVREs |
|
* and TREs. We use last processed offset to figure out which TRE is |
|
* associated with which EVRE. If two TREs are consumed by HW, the EVREs |
|
* are in order in the event ring. |
|
* |
|
* This handler will do a one pass for consuming EVREs. Other EVREs may |
|
* be delivered while we are working. It will try to consume incoming |
|
* EVREs one more time and return. |
|
* |
|
* For unprocessed EVREs, hardware will trigger another interrupt until |
|
* all the interrupt bits are cleared. |
|
* |
|
* Hardware guarantees that by the time interrupt is observed, all data |
|
* transactions in flight are delivered to their respective places and |
|
* are visible to the CPU. |
|
* |
|
* On demand paging for IOMMU is only supported for PCIe via PRI |
|
* (Page Request Interface) not for HIDMA. All other hardware instances |
|
* including HIDMA work on pinned DMA addresses. |
|
* |
|
* HIDMA is not aware of IOMMU presence since it follows the DMA API. All |
|
* IOMMU latency will be built into the data movement time. By the time |
|
* interrupt happens, IOMMU lookups + data movement has already taken place. |
|
* |
|
* While the first read in a typical PCI endpoint ISR flushes all outstanding |
|
* requests traditionally to the destination, this concept does not apply |
|
* here for this HW. |
|
*/ |
|
static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause) |
|
{ |
|
unsigned long irqflags; |
|
|
|
if (cause & HIDMA_ERR_INT_MASK) { |
|
dev_err(lldev->dev, "error 0x%x, disabling...\n", |
|
cause); |
|
|
|
/* Clear out pending interrupts */ |
|
writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
|
|
|
/* No further submissions. */ |
|
hidma_ll_disable(lldev); |
|
|
|
/* Driver completes the txn and intimates the client.*/ |
|
hidma_cleanup_pending_tre(lldev, 0xFF, |
|
HIDMA_EVRE_STATUS_ERROR); |
|
|
|
return; |
|
} |
|
|
|
spin_lock_irqsave(&lldev->lock, irqflags); |
|
writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
|
spin_unlock_irqrestore(&lldev->lock, irqflags); |
|
|
|
/* |
|
* Fine tuned for this HW... |
|
* |
|
* This ISR has been designed for this particular hardware. Relaxed |
|
* read and write accessors are used for performance reasons due to |
|
* interrupt delivery guarantees. Do not copy this code blindly and |
|
* expect that to work. |
|
* |
|
* Try to consume as many EVREs as possible. |
|
*/ |
|
hidma_handle_tre_completion(lldev); |
|
} |
|
|
|
irqreturn_t hidma_ll_inthandler(int chirq, void *arg) |
|
{ |
|
struct hidma_lldev *lldev = arg; |
|
u32 status; |
|
u32 enable; |
|
u32 cause; |
|
|
|
status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); |
|
enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
|
cause = status & enable; |
|
|
|
while (cause) { |
|
hidma_ll_int_handler_internal(lldev, cause); |
|
|
|
/* |
|
* Another interrupt might have arrived while we are |
|
* processing this one. Read the new cause. |
|
*/ |
|
status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); |
|
enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
|
cause = status & enable; |
|
} |
|
|
|
return IRQ_HANDLED; |
|
} |
|
|
|
irqreturn_t hidma_ll_inthandler_msi(int chirq, void *arg, int cause) |
|
{ |
|
struct hidma_lldev *lldev = arg; |
|
|
|
hidma_ll_int_handler_internal(lldev, cause); |
|
return IRQ_HANDLED; |
|
} |
|
|
|
int hidma_ll_enable(struct hidma_lldev *lldev) |
|
{ |
|
u32 val; |
|
int ret; |
|
|
|
val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
|
val &= ~(HIDMA_CH_CONTROL_MASK << 16); |
|
val |= HIDMA_CH_ENABLE << 16; |
|
writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
|
|
|
ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, |
|
hidma_is_chan_enabled(HIDMA_CH_STATE(val)), |
|
1000, 10000); |
|
if (ret) { |
|
dev_err(lldev->dev, "event channel did not get enabled\n"); |
|
return ret; |
|
} |
|
|
|
val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
|
val &= ~(HIDMA_CH_CONTROL_MASK << 16); |
|
val |= HIDMA_CH_ENABLE << 16; |
|
writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
|
|
|
ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, |
|
hidma_is_chan_enabled(HIDMA_CH_STATE(val)), |
|
1000, 10000); |
|
if (ret) { |
|
dev_err(lldev->dev, "transfer channel did not get enabled\n"); |
|
return ret; |
|
} |
|
|
|
lldev->trch_state = HIDMA_CH_ENABLED; |
|
lldev->evch_state = HIDMA_CH_ENABLED; |
|
|
|
/* enable irqs */ |
|
writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
|
|
|
return 0; |
|
} |
|
|
|
void hidma_ll_start(struct hidma_lldev *lldev) |
|
{ |
|
unsigned long irqflags; |
|
|
|
spin_lock_irqsave(&lldev->lock, irqflags); |
|
writel(lldev->tre_write_offset, lldev->trca + HIDMA_TRCA_DOORBELL_REG); |
|
spin_unlock_irqrestore(&lldev->lock, irqflags); |
|
} |
|
|
|
bool hidma_ll_isenabled(struct hidma_lldev *lldev) |
|
{ |
|
u32 val; |
|
|
|
val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
|
lldev->trch_state = HIDMA_CH_STATE(val); |
|
val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
|
lldev->evch_state = HIDMA_CH_STATE(val); |
|
|
|
/* both channels have to be enabled before calling this function */ |
|
if (hidma_is_chan_enabled(lldev->trch_state) && |
|
hidma_is_chan_enabled(lldev->evch_state)) |
|
return true; |
|
|
|
return false; |
|
} |
|
|
|
void hidma_ll_queue_request(struct hidma_lldev *lldev, u32 tre_ch) |
|
{ |
|
struct hidma_tre *tre; |
|
unsigned long flags; |
|
|
|
tre = &lldev->trepool[tre_ch]; |
|
|
|
/* copy the TRE into its location in the TRE ring */ |
|
spin_lock_irqsave(&lldev->lock, flags); |
|
tre->tre_index = lldev->tre_write_offset / HIDMA_TRE_SIZE; |
|
lldev->pending_tre_list[tre->tre_index] = tre; |
|
memcpy(lldev->tre_ring + lldev->tre_write_offset, |
|
&tre->tre_local[0], HIDMA_TRE_SIZE); |
|
tre->err_code = 0; |
|
tre->err_info = 0; |
|
tre->queued = 1; |
|
atomic_inc(&lldev->pending_tre_count); |
|
lldev->tre_write_offset = (lldev->tre_write_offset + HIDMA_TRE_SIZE) |
|
% lldev->tre_ring_size; |
|
spin_unlock_irqrestore(&lldev->lock, flags); |
|
} |
|
|
|
/* |
|
* Note that even though we stop this channel if there is a pending transaction |
|
* in flight it will complete and follow the callback. This request will |
|
* prevent further requests to be made. |
|
*/ |
|
int hidma_ll_disable(struct hidma_lldev *lldev) |
|
{ |
|
u32 val; |
|
int ret; |
|
|
|
/* The channel needs to be in working state */ |
|
if (!hidma_ll_isenabled(lldev)) |
|
return 0; |
|
|
|
val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
|
val &= ~(HIDMA_CH_CONTROL_MASK << 16); |
|
val |= HIDMA_CH_SUSPEND << 16; |
|
writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG); |
|
|
|
/* |
|
* Start the wait right after the suspend is confirmed. |
|
* Do a polled read up to 1ms and 10ms maximum. |
|
*/ |
|
ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, |
|
HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED, |
|
1000, 10000); |
|
if (ret) |
|
return ret; |
|
|
|
val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
|
val &= ~(HIDMA_CH_CONTROL_MASK << 16); |
|
val |= HIDMA_CH_SUSPEND << 16; |
|
writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG); |
|
|
|
/* |
|
* Start the wait right after the suspend is confirmed |
|
* Delay up to 10ms after reset to allow DMA logic to quiesce. |
|
*/ |
|
ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, |
|
HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED, |
|
1000, 10000); |
|
if (ret) |
|
return ret; |
|
|
|
lldev->trch_state = HIDMA_CH_SUSPENDED; |
|
lldev->evch_state = HIDMA_CH_SUSPENDED; |
|
|
|
/* disable interrupts */ |
|
writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
|
return 0; |
|
} |
|
|
|
void hidma_ll_set_transfer_params(struct hidma_lldev *lldev, u32 tre_ch, |
|
dma_addr_t src, dma_addr_t dest, u32 len, |
|
u32 flags, u32 txntype) |
|
{ |
|
struct hidma_tre *tre; |
|
u32 *tre_local; |
|
|
|
if (tre_ch >= lldev->nr_tres) { |
|
dev_err(lldev->dev, "invalid TRE number in transfer params:%d", |
|
tre_ch); |
|
return; |
|
} |
|
|
|
tre = &lldev->trepool[tre_ch]; |
|
if (atomic_read(&tre->allocated) != true) { |
|
dev_err(lldev->dev, "trying to set params on an unused TRE:%d", |
|
tre_ch); |
|
return; |
|
} |
|
|
|
tre_local = &tre->tre_local[0]; |
|
tre_local[HIDMA_TRE_CFG_IDX] &= ~GENMASK(7, 0); |
|
tre_local[HIDMA_TRE_CFG_IDX] |= txntype; |
|
tre_local[HIDMA_TRE_LEN_IDX] = len; |
|
tre_local[HIDMA_TRE_SRC_LOW_IDX] = lower_32_bits(src); |
|
tre_local[HIDMA_TRE_SRC_HI_IDX] = upper_32_bits(src); |
|
tre_local[HIDMA_TRE_DEST_LOW_IDX] = lower_32_bits(dest); |
|
tre_local[HIDMA_TRE_DEST_HI_IDX] = upper_32_bits(dest); |
|
tre->int_flags = flags; |
|
} |
|
|
|
/* |
|
* Called during initialization and after an error condition |
|
* to restore hardware state. |
|
*/ |
|
int hidma_ll_setup(struct hidma_lldev *lldev) |
|
{ |
|
int rc; |
|
u64 addr; |
|
u32 val; |
|
u32 nr_tres = lldev->nr_tres; |
|
|
|
atomic_set(&lldev->pending_tre_count, 0); |
|
lldev->tre_processed_off = 0; |
|
lldev->evre_processed_off = 0; |
|
lldev->tre_write_offset = 0; |
|
|
|
/* disable interrupts */ |
|
writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
|
|
|
/* clear all pending interrupts */ |
|
val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); |
|
writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
|
|
|
rc = hidma_ll_reset(lldev); |
|
if (rc) |
|
return rc; |
|
|
|
/* |
|
* Clear all pending interrupts again. |
|
* Otherwise, we observe reset complete interrupts. |
|
*/ |
|
val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); |
|
writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
|
|
|
/* disable interrupts again after reset */ |
|
writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
|
|
|
addr = lldev->tre_dma; |
|
writel(lower_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_LOW_REG); |
|
writel(upper_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_HIGH_REG); |
|
writel(lldev->tre_ring_size, lldev->trca + HIDMA_TRCA_RING_LEN_REG); |
|
|
|
addr = lldev->evre_dma; |
|
writel(lower_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_LOW_REG); |
|
writel(upper_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_HIGH_REG); |
|
writel(HIDMA_EVRE_SIZE * nr_tres, |
|
lldev->evca + HIDMA_EVCA_RING_LEN_REG); |
|
|
|
/* configure interrupts */ |
|
hidma_ll_setup_irq(lldev, lldev->msi_support); |
|
|
|
rc = hidma_ll_enable(lldev); |
|
if (rc) |
|
return rc; |
|
|
|
return rc; |
|
} |
|
|
|
void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi) |
|
{ |
|
u32 val; |
|
|
|
lldev->msi_support = msi; |
|
|
|
/* disable interrupts again after reset */ |
|
writel(0, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
|
writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
|
|
|
/* support IRQ by default */ |
|
val = readl(lldev->evca + HIDMA_EVCA_INTCTRL_REG); |
|
val &= ~0xF; |
|
if (!lldev->msi_support) |
|
val = val | 0x1; |
|
writel(val, lldev->evca + HIDMA_EVCA_INTCTRL_REG); |
|
|
|
/* clear all pending interrupts and enable them */ |
|
writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
|
writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
|
} |
|
|
|
struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres, |
|
void __iomem *trca, void __iomem *evca, |
|
u8 chidx) |
|
{ |
|
u32 required_bytes; |
|
struct hidma_lldev *lldev; |
|
int rc; |
|
size_t sz; |
|
|
|
if (!trca || !evca || !dev || !nr_tres) |
|
return NULL; |
|
|
|
/* need at least four TREs */ |
|
if (nr_tres < 4) |
|
return NULL; |
|
|
|
/* need an extra space */ |
|
nr_tres += 1; |
|
|
|
lldev = devm_kzalloc(dev, sizeof(struct hidma_lldev), GFP_KERNEL); |
|
if (!lldev) |
|
return NULL; |
|
|
|
lldev->evca = evca; |
|
lldev->trca = trca; |
|
lldev->dev = dev; |
|
sz = sizeof(struct hidma_tre); |
|
lldev->trepool = devm_kcalloc(lldev->dev, nr_tres, sz, GFP_KERNEL); |
|
if (!lldev->trepool) |
|
return NULL; |
|
|
|
required_bytes = sizeof(lldev->pending_tre_list[0]); |
|
lldev->pending_tre_list = devm_kcalloc(dev, nr_tres, required_bytes, |
|
GFP_KERNEL); |
|
if (!lldev->pending_tre_list) |
|
return NULL; |
|
|
|
sz = (HIDMA_TRE_SIZE + 1) * nr_tres; |
|
lldev->tre_ring = dmam_alloc_coherent(dev, sz, &lldev->tre_dma, |
|
GFP_KERNEL); |
|
if (!lldev->tre_ring) |
|
return NULL; |
|
|
|
lldev->tre_ring_size = HIDMA_TRE_SIZE * nr_tres; |
|
lldev->nr_tres = nr_tres; |
|
|
|
/* the TRE ring has to be TRE_SIZE aligned */ |
|
if (!IS_ALIGNED(lldev->tre_dma, HIDMA_TRE_SIZE)) { |
|
u8 tre_ring_shift; |
|
|
|
tre_ring_shift = lldev->tre_dma % HIDMA_TRE_SIZE; |
|
tre_ring_shift = HIDMA_TRE_SIZE - tre_ring_shift; |
|
lldev->tre_dma += tre_ring_shift; |
|
lldev->tre_ring += tre_ring_shift; |
|
} |
|
|
|
sz = (HIDMA_EVRE_SIZE + 1) * nr_tres; |
|
lldev->evre_ring = dmam_alloc_coherent(dev, sz, &lldev->evre_dma, |
|
GFP_KERNEL); |
|
if (!lldev->evre_ring) |
|
return NULL; |
|
|
|
lldev->evre_ring_size = HIDMA_EVRE_SIZE * nr_tres; |
|
|
|
/* the EVRE ring has to be EVRE_SIZE aligned */ |
|
if (!IS_ALIGNED(lldev->evre_dma, HIDMA_EVRE_SIZE)) { |
|
u8 evre_ring_shift; |
|
|
|
evre_ring_shift = lldev->evre_dma % HIDMA_EVRE_SIZE; |
|
evre_ring_shift = HIDMA_EVRE_SIZE - evre_ring_shift; |
|
lldev->evre_dma += evre_ring_shift; |
|
lldev->evre_ring += evre_ring_shift; |
|
} |
|
lldev->nr_tres = nr_tres; |
|
lldev->chidx = chidx; |
|
|
|
sz = nr_tres * sizeof(struct hidma_tre *); |
|
rc = kfifo_alloc(&lldev->handoff_fifo, sz, GFP_KERNEL); |
|
if (rc) |
|
return NULL; |
|
|
|
rc = hidma_ll_setup(lldev); |
|
if (rc) |
|
return NULL; |
|
|
|
spin_lock_init(&lldev->lock); |
|
tasklet_setup(&lldev->task, hidma_ll_tre_complete); |
|
lldev->initialized = 1; |
|
writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
|
return lldev; |
|
} |
|
|
|
int hidma_ll_uninit(struct hidma_lldev *lldev) |
|
{ |
|
u32 required_bytes; |
|
int rc = 0; |
|
u32 val; |
|
|
|
if (!lldev) |
|
return -ENODEV; |
|
|
|
if (!lldev->initialized) |
|
return 0; |
|
|
|
lldev->initialized = 0; |
|
|
|
required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres; |
|
tasklet_kill(&lldev->task); |
|
memset(lldev->trepool, 0, required_bytes); |
|
lldev->trepool = NULL; |
|
atomic_set(&lldev->pending_tre_count, 0); |
|
lldev->tre_write_offset = 0; |
|
|
|
rc = hidma_ll_reset(lldev); |
|
|
|
/* |
|
* Clear all pending interrupts again. |
|
* Otherwise, we observe reset complete interrupts. |
|
*/ |
|
val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); |
|
writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); |
|
writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); |
|
return rc; |
|
} |
|
|
|
enum dma_status hidma_ll_status(struct hidma_lldev *lldev, u32 tre_ch) |
|
{ |
|
enum dma_status ret = DMA_ERROR; |
|
struct hidma_tre *tre; |
|
unsigned long flags; |
|
u8 err_code; |
|
|
|
spin_lock_irqsave(&lldev->lock, flags); |
|
|
|
tre = &lldev->trepool[tre_ch]; |
|
err_code = tre->err_code; |
|
|
|
if (err_code & HIDMA_EVRE_STATUS_COMPLETE) |
|
ret = DMA_COMPLETE; |
|
else if (err_code & HIDMA_EVRE_STATUS_ERROR) |
|
ret = DMA_ERROR; |
|
else |
|
ret = DMA_IN_PROGRESS; |
|
spin_unlock_irqrestore(&lldev->lock, flags); |
|
|
|
return ret; |
|
}
|
|
|