forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
758 lines
18 KiB
758 lines
18 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* |
|
* AMD Cryptographic Coprocessor (CCP) driver |
|
* |
|
* Copyright (C) 2016,2019 Advanced Micro Devices, Inc. |
|
* |
|
* Author: Gary R Hook <[email protected]> |
|
*/ |
|
|
|
#include <linux/module.h> |
|
#include <linux/kernel.h> |
|
#include <linux/dma-mapping.h> |
|
#include <linux/dmaengine.h> |
|
#include <linux/spinlock.h> |
|
#include <linux/mutex.h> |
|
#include <linux/ccp.h> |
|
|
|
#include "ccp-dev.h" |
|
#include "../../dma/dmaengine.h" |
|
|
|
#define CCP_DMA_WIDTH(_mask) \ |
|
({ \ |
|
u64 mask = _mask + 1; \ |
|
(mask == 0) ? 64 : fls64(mask); \ |
|
}) |
|
|
|
/* The CCP as a DMA provider can be configured for public or private |
|
* channels. Default is specified in the vdata for the device (PCI ID). |
|
* This module parameter will override for all channels on all devices: |
|
* dma_chan_attr = 0x2 to force all channels public |
|
* = 0x1 to force all channels private |
|
* = 0x0 to defer to the vdata setting |
|
* = any other value: warning, revert to 0x0 |
|
*/ |
|
static unsigned int dma_chan_attr = CCP_DMA_DFLT; |
|
module_param(dma_chan_attr, uint, 0444); |
|
MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public"); |
|
|
|
static unsigned int dmaengine = 1; |
|
module_param(dmaengine, uint, 0444); |
|
MODULE_PARM_DESC(dmaengine, "Register services with the DMA subsystem (any non-zero value, default: 1)"); |
|
|
|
static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp) |
|
{ |
|
switch (dma_chan_attr) { |
|
case CCP_DMA_DFLT: |
|
return ccp->vdata->dma_chan_attr; |
|
|
|
case CCP_DMA_PRIV: |
|
return DMA_PRIVATE; |
|
|
|
case CCP_DMA_PUB: |
|
return 0; |
|
|
|
default: |
|
dev_info_once(ccp->dev, "Invalid value for dma_chan_attr: %d\n", |
|
dma_chan_attr); |
|
return ccp->vdata->dma_chan_attr; |
|
} |
|
} |
|
|
|
static void ccp_free_cmd_resources(struct ccp_device *ccp, |
|
struct list_head *list) |
|
{ |
|
struct ccp_dma_cmd *cmd, *ctmp; |
|
|
|
list_for_each_entry_safe(cmd, ctmp, list, entry) { |
|
list_del(&cmd->entry); |
|
kmem_cache_free(ccp->dma_cmd_cache, cmd); |
|
} |
|
} |
|
|
|
static void ccp_free_desc_resources(struct ccp_device *ccp, |
|
struct list_head *list) |
|
{ |
|
struct ccp_dma_desc *desc, *dtmp; |
|
|
|
list_for_each_entry_safe(desc, dtmp, list, entry) { |
|
ccp_free_cmd_resources(ccp, &desc->active); |
|
ccp_free_cmd_resources(ccp, &desc->pending); |
|
|
|
list_del(&desc->entry); |
|
kmem_cache_free(ccp->dma_desc_cache, desc); |
|
} |
|
} |
|
|
|
static void ccp_free_chan_resources(struct dma_chan *dma_chan) |
|
{ |
|
struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, |
|
dma_chan); |
|
unsigned long flags; |
|
|
|
dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan); |
|
|
|
spin_lock_irqsave(&chan->lock, flags); |
|
|
|
ccp_free_desc_resources(chan->ccp, &chan->complete); |
|
ccp_free_desc_resources(chan->ccp, &chan->active); |
|
ccp_free_desc_resources(chan->ccp, &chan->pending); |
|
ccp_free_desc_resources(chan->ccp, &chan->created); |
|
|
|
spin_unlock_irqrestore(&chan->lock, flags); |
|
} |
|
|
|
static void ccp_cleanup_desc_resources(struct ccp_device *ccp, |
|
struct list_head *list) |
|
{ |
|
struct ccp_dma_desc *desc, *dtmp; |
|
|
|
list_for_each_entry_safe_reverse(desc, dtmp, list, entry) { |
|
if (!async_tx_test_ack(&desc->tx_desc)) |
|
continue; |
|
|
|
dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); |
|
|
|
ccp_free_cmd_resources(ccp, &desc->active); |
|
ccp_free_cmd_resources(ccp, &desc->pending); |
|
|
|
list_del(&desc->entry); |
|
kmem_cache_free(ccp->dma_desc_cache, desc); |
|
} |
|
} |
|
|
|
static void ccp_do_cleanup(unsigned long data) |
|
{ |
|
struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data; |
|
unsigned long flags; |
|
|
|
dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__, |
|
dma_chan_name(&chan->dma_chan)); |
|
|
|
spin_lock_irqsave(&chan->lock, flags); |
|
|
|
ccp_cleanup_desc_resources(chan->ccp, &chan->complete); |
|
|
|
spin_unlock_irqrestore(&chan->lock, flags); |
|
} |
|
|
|
static int ccp_issue_next_cmd(struct ccp_dma_desc *desc) |
|
{ |
|
struct ccp_dma_cmd *cmd; |
|
int ret; |
|
|
|
cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry); |
|
list_move(&cmd->entry, &desc->active); |
|
|
|
dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__, |
|
desc->tx_desc.cookie, cmd); |
|
|
|
ret = ccp_enqueue_cmd(&cmd->ccp_cmd); |
|
if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY)) |
|
return 0; |
|
|
|
dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__, |
|
ret, desc->tx_desc.cookie, cmd); |
|
|
|
return ret; |
|
} |
|
|
|
static void ccp_free_active_cmd(struct ccp_dma_desc *desc) |
|
{ |
|
struct ccp_dma_cmd *cmd; |
|
|
|
cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd, |
|
entry); |
|
if (!cmd) |
|
return; |
|
|
|
dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n", |
|
__func__, desc->tx_desc.cookie, cmd); |
|
|
|
list_del(&cmd->entry); |
|
kmem_cache_free(desc->ccp->dma_cmd_cache, cmd); |
|
} |
|
|
|
static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan, |
|
struct ccp_dma_desc *desc) |
|
{ |
|
/* Move current DMA descriptor to the complete list */ |
|
if (desc) |
|
list_move(&desc->entry, &chan->complete); |
|
|
|
/* Get the next DMA descriptor on the active list */ |
|
desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, |
|
entry); |
|
|
|
return desc; |
|
} |
|
|
|
static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan, |
|
struct ccp_dma_desc *desc) |
|
{ |
|
struct dma_async_tx_descriptor *tx_desc; |
|
unsigned long flags; |
|
|
|
/* Loop over descriptors until one is found with commands */ |
|
do { |
|
if (desc) { |
|
/* Remove the DMA command from the list and free it */ |
|
ccp_free_active_cmd(desc); |
|
|
|
if (!list_empty(&desc->pending)) { |
|
/* No errors, keep going */ |
|
if (desc->status != DMA_ERROR) |
|
return desc; |
|
|
|
/* Error, free remaining commands and move on */ |
|
ccp_free_cmd_resources(desc->ccp, |
|
&desc->pending); |
|
} |
|
|
|
tx_desc = &desc->tx_desc; |
|
} else { |
|
tx_desc = NULL; |
|
} |
|
|
|
spin_lock_irqsave(&chan->lock, flags); |
|
|
|
if (desc) { |
|
if (desc->status != DMA_ERROR) |
|
desc->status = DMA_COMPLETE; |
|
|
|
dev_dbg(desc->ccp->dev, |
|
"%s - tx %d complete, status=%u\n", __func__, |
|
desc->tx_desc.cookie, desc->status); |
|
|
|
dma_cookie_complete(tx_desc); |
|
dma_descriptor_unmap(tx_desc); |
|
} |
|
|
|
desc = __ccp_next_dma_desc(chan, desc); |
|
|
|
spin_unlock_irqrestore(&chan->lock, flags); |
|
|
|
if (tx_desc) { |
|
dmaengine_desc_get_callback_invoke(tx_desc, NULL); |
|
|
|
dma_run_dependencies(tx_desc); |
|
} |
|
} while (desc); |
|
|
|
return NULL; |
|
} |
|
|
|
static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan) |
|
{ |
|
struct ccp_dma_desc *desc; |
|
|
|
if (list_empty(&chan->pending)) |
|
return NULL; |
|
|
|
desc = list_empty(&chan->active) |
|
? list_first_entry(&chan->pending, struct ccp_dma_desc, entry) |
|
: NULL; |
|
|
|
list_splice_tail_init(&chan->pending, &chan->active); |
|
|
|
return desc; |
|
} |
|
|
|
static void ccp_cmd_callback(void *data, int err) |
|
{ |
|
struct ccp_dma_desc *desc = data; |
|
struct ccp_dma_chan *chan; |
|
int ret; |
|
|
|
if (err == -EINPROGRESS) |
|
return; |
|
|
|
chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan, |
|
dma_chan); |
|
|
|
dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n", |
|
__func__, desc->tx_desc.cookie, err); |
|
|
|
if (err) |
|
desc->status = DMA_ERROR; |
|
|
|
while (true) { |
|
/* Check for DMA descriptor completion */ |
|
desc = ccp_handle_active_desc(chan, desc); |
|
|
|
/* Don't submit cmd if no descriptor or DMA is paused */ |
|
if (!desc || (chan->status == DMA_PAUSED)) |
|
break; |
|
|
|
ret = ccp_issue_next_cmd(desc); |
|
if (!ret) |
|
break; |
|
|
|
desc->status = DMA_ERROR; |
|
} |
|
|
|
tasklet_schedule(&chan->cleanup_tasklet); |
|
} |
|
|
|
static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc) |
|
{ |
|
struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc, |
|
tx_desc); |
|
struct ccp_dma_chan *chan; |
|
dma_cookie_t cookie; |
|
unsigned long flags; |
|
|
|
chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan); |
|
|
|
spin_lock_irqsave(&chan->lock, flags); |
|
|
|
cookie = dma_cookie_assign(tx_desc); |
|
list_move_tail(&desc->entry, &chan->pending); |
|
|
|
spin_unlock_irqrestore(&chan->lock, flags); |
|
|
|
dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n", |
|
__func__, cookie); |
|
|
|
return cookie; |
|
} |
|
|
|
static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan) |
|
{ |
|
struct ccp_dma_cmd *cmd; |
|
|
|
cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT); |
|
if (cmd) |
|
memset(cmd, 0, sizeof(*cmd)); |
|
|
|
return cmd; |
|
} |
|
|
|
static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan, |
|
unsigned long flags) |
|
{ |
|
struct ccp_dma_desc *desc; |
|
|
|
desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT); |
|
if (!desc) |
|
return NULL; |
|
|
|
dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan); |
|
desc->tx_desc.flags = flags; |
|
desc->tx_desc.tx_submit = ccp_tx_submit; |
|
desc->ccp = chan->ccp; |
|
INIT_LIST_HEAD(&desc->entry); |
|
INIT_LIST_HEAD(&desc->pending); |
|
INIT_LIST_HEAD(&desc->active); |
|
desc->status = DMA_IN_PROGRESS; |
|
|
|
return desc; |
|
} |
|
|
|
static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan, |
|
struct scatterlist *dst_sg, |
|
unsigned int dst_nents, |
|
struct scatterlist *src_sg, |
|
unsigned int src_nents, |
|
unsigned long flags) |
|
{ |
|
struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, |
|
dma_chan); |
|
struct ccp_device *ccp = chan->ccp; |
|
struct ccp_dma_desc *desc; |
|
struct ccp_dma_cmd *cmd; |
|
struct ccp_cmd *ccp_cmd; |
|
struct ccp_passthru_nomap_engine *ccp_pt; |
|
unsigned int src_offset, src_len; |
|
unsigned int dst_offset, dst_len; |
|
unsigned int len; |
|
unsigned long sflags; |
|
size_t total_len; |
|
|
|
if (!dst_sg || !src_sg) |
|
return NULL; |
|
|
|
if (!dst_nents || !src_nents) |
|
return NULL; |
|
|
|
desc = ccp_alloc_dma_desc(chan, flags); |
|
if (!desc) |
|
return NULL; |
|
|
|
total_len = 0; |
|
|
|
src_len = sg_dma_len(src_sg); |
|
src_offset = 0; |
|
|
|
dst_len = sg_dma_len(dst_sg); |
|
dst_offset = 0; |
|
|
|
while (true) { |
|
if (!src_len) { |
|
src_nents--; |
|
if (!src_nents) |
|
break; |
|
|
|
src_sg = sg_next(src_sg); |
|
if (!src_sg) |
|
break; |
|
|
|
src_len = sg_dma_len(src_sg); |
|
src_offset = 0; |
|
continue; |
|
} |
|
|
|
if (!dst_len) { |
|
dst_nents--; |
|
if (!dst_nents) |
|
break; |
|
|
|
dst_sg = sg_next(dst_sg); |
|
if (!dst_sg) |
|
break; |
|
|
|
dst_len = sg_dma_len(dst_sg); |
|
dst_offset = 0; |
|
continue; |
|
} |
|
|
|
len = min(dst_len, src_len); |
|
|
|
cmd = ccp_alloc_dma_cmd(chan); |
|
if (!cmd) |
|
goto err; |
|
|
|
ccp_cmd = &cmd->ccp_cmd; |
|
ccp_cmd->ccp = chan->ccp; |
|
ccp_pt = &ccp_cmd->u.passthru_nomap; |
|
ccp_cmd->flags = CCP_CMD_MAY_BACKLOG; |
|
ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP; |
|
ccp_cmd->engine = CCP_ENGINE_PASSTHRU; |
|
ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP; |
|
ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP; |
|
ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset; |
|
ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset; |
|
ccp_pt->src_len = len; |
|
ccp_pt->final = 1; |
|
ccp_cmd->callback = ccp_cmd_callback; |
|
ccp_cmd->data = desc; |
|
|
|
list_add_tail(&cmd->entry, &desc->pending); |
|
|
|
dev_dbg(ccp->dev, |
|
"%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__, |
|
cmd, &ccp_pt->src_dma, |
|
&ccp_pt->dst_dma, ccp_pt->src_len); |
|
|
|
total_len += len; |
|
|
|
src_len -= len; |
|
src_offset += len; |
|
|
|
dst_len -= len; |
|
dst_offset += len; |
|
} |
|
|
|
desc->len = total_len; |
|
|
|
if (list_empty(&desc->pending)) |
|
goto err; |
|
|
|
dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); |
|
|
|
spin_lock_irqsave(&chan->lock, sflags); |
|
|
|
list_add_tail(&desc->entry, &chan->created); |
|
|
|
spin_unlock_irqrestore(&chan->lock, sflags); |
|
|
|
return desc; |
|
|
|
err: |
|
ccp_free_cmd_resources(ccp, &desc->pending); |
|
kmem_cache_free(ccp->dma_desc_cache, desc); |
|
|
|
return NULL; |
|
} |
|
|
|
static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy( |
|
struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len, |
|
unsigned long flags) |
|
{ |
|
struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, |
|
dma_chan); |
|
struct ccp_dma_desc *desc; |
|
struct scatterlist dst_sg, src_sg; |
|
|
|
dev_dbg(chan->ccp->dev, |
|
"%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n", |
|
__func__, &src, &dst, len, flags); |
|
|
|
sg_init_table(&dst_sg, 1); |
|
sg_dma_address(&dst_sg) = dst; |
|
sg_dma_len(&dst_sg) = len; |
|
|
|
sg_init_table(&src_sg, 1); |
|
sg_dma_address(&src_sg) = src; |
|
sg_dma_len(&src_sg) = len; |
|
|
|
desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags); |
|
if (!desc) |
|
return NULL; |
|
|
|
return &desc->tx_desc; |
|
} |
|
|
|
static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt( |
|
struct dma_chan *dma_chan, unsigned long flags) |
|
{ |
|
struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, |
|
dma_chan); |
|
struct ccp_dma_desc *desc; |
|
|
|
desc = ccp_alloc_dma_desc(chan, flags); |
|
if (!desc) |
|
return NULL; |
|
|
|
return &desc->tx_desc; |
|
} |
|
|
|
static void ccp_issue_pending(struct dma_chan *dma_chan) |
|
{ |
|
struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, |
|
dma_chan); |
|
struct ccp_dma_desc *desc; |
|
unsigned long flags; |
|
|
|
dev_dbg(chan->ccp->dev, "%s\n", __func__); |
|
|
|
spin_lock_irqsave(&chan->lock, flags); |
|
|
|
desc = __ccp_pending_to_active(chan); |
|
|
|
spin_unlock_irqrestore(&chan->lock, flags); |
|
|
|
/* If there was nothing active, start processing */ |
|
if (desc) |
|
ccp_cmd_callback(desc, 0); |
|
} |
|
|
|
static enum dma_status ccp_tx_status(struct dma_chan *dma_chan, |
|
dma_cookie_t cookie, |
|
struct dma_tx_state *state) |
|
{ |
|
struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, |
|
dma_chan); |
|
struct ccp_dma_desc *desc; |
|
enum dma_status ret; |
|
unsigned long flags; |
|
|
|
if (chan->status == DMA_PAUSED) { |
|
ret = DMA_PAUSED; |
|
goto out; |
|
} |
|
|
|
ret = dma_cookie_status(dma_chan, cookie, state); |
|
if (ret == DMA_COMPLETE) { |
|
spin_lock_irqsave(&chan->lock, flags); |
|
|
|
/* Get status from complete chain, if still there */ |
|
list_for_each_entry(desc, &chan->complete, entry) { |
|
if (desc->tx_desc.cookie != cookie) |
|
continue; |
|
|
|
ret = desc->status; |
|
break; |
|
} |
|
|
|
spin_unlock_irqrestore(&chan->lock, flags); |
|
} |
|
|
|
out: |
|
dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret); |
|
|
|
return ret; |
|
} |
|
|
|
static int ccp_pause(struct dma_chan *dma_chan) |
|
{ |
|
struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, |
|
dma_chan); |
|
|
|
chan->status = DMA_PAUSED; |
|
|
|
/*TODO: Wait for active DMA to complete before returning? */ |
|
|
|
return 0; |
|
} |
|
|
|
static int ccp_resume(struct dma_chan *dma_chan) |
|
{ |
|
struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, |
|
dma_chan); |
|
struct ccp_dma_desc *desc; |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&chan->lock, flags); |
|
|
|
desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, |
|
entry); |
|
|
|
spin_unlock_irqrestore(&chan->lock, flags); |
|
|
|
/* Indicate the channel is running again */ |
|
chan->status = DMA_IN_PROGRESS; |
|
|
|
/* If there was something active, re-start */ |
|
if (desc) |
|
ccp_cmd_callback(desc, 0); |
|
|
|
return 0; |
|
} |
|
|
|
static int ccp_terminate_all(struct dma_chan *dma_chan) |
|
{ |
|
struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, |
|
dma_chan); |
|
unsigned long flags; |
|
|
|
dev_dbg(chan->ccp->dev, "%s\n", __func__); |
|
|
|
/*TODO: Wait for active DMA to complete before continuing */ |
|
|
|
spin_lock_irqsave(&chan->lock, flags); |
|
|
|
/*TODO: Purge the complete list? */ |
|
ccp_free_desc_resources(chan->ccp, &chan->active); |
|
ccp_free_desc_resources(chan->ccp, &chan->pending); |
|
ccp_free_desc_resources(chan->ccp, &chan->created); |
|
|
|
spin_unlock_irqrestore(&chan->lock, flags); |
|
|
|
return 0; |
|
} |
|
|
|
int ccp_dmaengine_register(struct ccp_device *ccp) |
|
{ |
|
struct ccp_dma_chan *chan; |
|
struct dma_device *dma_dev = &ccp->dma_dev; |
|
struct dma_chan *dma_chan; |
|
char *dma_cmd_cache_name; |
|
char *dma_desc_cache_name; |
|
unsigned int i; |
|
int ret; |
|
|
|
if (!dmaengine) |
|
return 0; |
|
|
|
ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count, |
|
sizeof(*(ccp->ccp_dma_chan)), |
|
GFP_KERNEL); |
|
if (!ccp->ccp_dma_chan) |
|
return -ENOMEM; |
|
|
|
dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, |
|
"%s-dmaengine-cmd-cache", |
|
ccp->name); |
|
if (!dma_cmd_cache_name) |
|
return -ENOMEM; |
|
|
|
ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name, |
|
sizeof(struct ccp_dma_cmd), |
|
sizeof(void *), |
|
SLAB_HWCACHE_ALIGN, NULL); |
|
if (!ccp->dma_cmd_cache) |
|
return -ENOMEM; |
|
|
|
dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, |
|
"%s-dmaengine-desc-cache", |
|
ccp->name); |
|
if (!dma_desc_cache_name) { |
|
ret = -ENOMEM; |
|
goto err_cache; |
|
} |
|
|
|
ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name, |
|
sizeof(struct ccp_dma_desc), |
|
sizeof(void *), |
|
SLAB_HWCACHE_ALIGN, NULL); |
|
if (!ccp->dma_desc_cache) { |
|
ret = -ENOMEM; |
|
goto err_cache; |
|
} |
|
|
|
dma_dev->dev = ccp->dev; |
|
dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); |
|
dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); |
|
dma_dev->directions = DMA_MEM_TO_MEM; |
|
dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; |
|
dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); |
|
dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); |
|
|
|
/* The DMA channels for this device can be set to public or private, |
|
* and overridden by the module parameter dma_chan_attr. |
|
* Default: according to the value in vdata (dma_chan_attr=0) |
|
* dma_chan_attr=0x1: all channels private (override vdata) |
|
* dma_chan_attr=0x2: all channels public (override vdata) |
|
*/ |
|
if (ccp_get_dma_chan_attr(ccp) == DMA_PRIVATE) |
|
dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); |
|
|
|
INIT_LIST_HEAD(&dma_dev->channels); |
|
for (i = 0; i < ccp->cmd_q_count; i++) { |
|
chan = ccp->ccp_dma_chan + i; |
|
dma_chan = &chan->dma_chan; |
|
|
|
chan->ccp = ccp; |
|
|
|
spin_lock_init(&chan->lock); |
|
INIT_LIST_HEAD(&chan->created); |
|
INIT_LIST_HEAD(&chan->pending); |
|
INIT_LIST_HEAD(&chan->active); |
|
INIT_LIST_HEAD(&chan->complete); |
|
|
|
tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup, |
|
(unsigned long)chan); |
|
|
|
dma_chan->device = dma_dev; |
|
dma_cookie_init(dma_chan); |
|
|
|
list_add_tail(&dma_chan->device_node, &dma_dev->channels); |
|
} |
|
|
|
dma_dev->device_free_chan_resources = ccp_free_chan_resources; |
|
dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy; |
|
dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt; |
|
dma_dev->device_issue_pending = ccp_issue_pending; |
|
dma_dev->device_tx_status = ccp_tx_status; |
|
dma_dev->device_pause = ccp_pause; |
|
dma_dev->device_resume = ccp_resume; |
|
dma_dev->device_terminate_all = ccp_terminate_all; |
|
|
|
ret = dma_async_device_register(dma_dev); |
|
if (ret) |
|
goto err_reg; |
|
|
|
return 0; |
|
|
|
err_reg: |
|
kmem_cache_destroy(ccp->dma_desc_cache); |
|
|
|
err_cache: |
|
kmem_cache_destroy(ccp->dma_cmd_cache); |
|
|
|
return ret; |
|
} |
|
|
|
void ccp_dmaengine_unregister(struct ccp_device *ccp) |
|
{ |
|
struct dma_device *dma_dev = &ccp->dma_dev; |
|
|
|
if (!dmaengine) |
|
return; |
|
|
|
dma_async_device_unregister(dma_dev); |
|
|
|
kmem_cache_destroy(ccp->dma_desc_cache); |
|
kmem_cache_destroy(ccp->dma_cmd_cache); |
|
}
|
|
|