forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
544 lines
15 KiB
544 lines
15 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* Copyright (C) 2020 Marvell. */ |
|
|
|
#include "otx2_cptvf.h" |
|
#include "otx2_cpt_common.h" |
|
|
|
/* SG list header size in bytes */ |
|
#define SG_LIST_HDR_SIZE 8 |
|
|
|
/* Default timeout when waiting for free pending entry in us */ |
|
#define CPT_PENTRY_TIMEOUT 1000 |
|
#define CPT_PENTRY_STEP 50 |
|
|
|
/* Default threshold for stopping and resuming sender requests */ |
|
#define CPT_IQ_STOP_MARGIN 128 |
|
#define CPT_IQ_RESUME_MARGIN 512 |
|
|
|
/* Default command timeout in seconds */ |
|
#define CPT_COMMAND_TIMEOUT 4 |
|
#define CPT_TIME_IN_RESET_COUNT 5 |
|
|
|
static void otx2_cpt_dump_sg_list(struct pci_dev *pdev, |
|
struct otx2_cpt_req_info *req) |
|
{ |
|
int i; |
|
|
|
pr_debug("Gather list size %d\n", req->in_cnt); |
|
for (i = 0; i < req->in_cnt; i++) { |
|
pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, |
|
req->in[i].size, req->in[i].vptr, |
|
(void *) req->in[i].dma_addr); |
|
pr_debug("Buffer hexdump (%d bytes)\n", |
|
req->in[i].size); |
|
print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, |
|
req->in[i].vptr, req->in[i].size, false); |
|
} |
|
pr_debug("Scatter list size %d\n", req->out_cnt); |
|
for (i = 0; i < req->out_cnt; i++) { |
|
pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, |
|
req->out[i].size, req->out[i].vptr, |
|
(void *) req->out[i].dma_addr); |
|
pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size); |
|
print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, |
|
req->out[i].vptr, req->out[i].size, false); |
|
} |
|
} |
|
|
|
static inline struct otx2_cpt_pending_entry *get_free_pending_entry( |
|
struct otx2_cpt_pending_queue *q, |
|
int qlen) |
|
{ |
|
struct otx2_cpt_pending_entry *ent = NULL; |
|
|
|
ent = &q->head[q->rear]; |
|
if (unlikely(ent->busy)) |
|
return NULL; |
|
|
|
q->rear++; |
|
if (unlikely(q->rear == qlen)) |
|
q->rear = 0; |
|
|
|
return ent; |
|
} |
|
|
|
static inline u32 modulo_inc(u32 index, u32 length, u32 inc) |
|
{ |
|
if (WARN_ON(inc > length)) |
|
inc = length; |
|
|
|
index += inc; |
|
if (unlikely(index >= length)) |
|
index -= length; |
|
|
|
return index; |
|
} |
|
|
|
static inline void free_pentry(struct otx2_cpt_pending_entry *pentry) |
|
{ |
|
pentry->completion_addr = NULL; |
|
pentry->info = NULL; |
|
pentry->callback = NULL; |
|
pentry->areq = NULL; |
|
pentry->resume_sender = false; |
|
pentry->busy = false; |
|
} |
|
|
|
static inline int setup_sgio_components(struct pci_dev *pdev, |
|
struct otx2_cpt_buf_ptr *list, |
|
int buf_count, u8 *buffer) |
|
{ |
|
struct otx2_cpt_sglist_component *sg_ptr = NULL; |
|
int ret = 0, i, j; |
|
int components; |
|
|
|
if (unlikely(!list)) { |
|
dev_err(&pdev->dev, "Input list pointer is NULL\n"); |
|
return -EFAULT; |
|
} |
|
|
|
for (i = 0; i < buf_count; i++) { |
|
if (unlikely(!list[i].vptr)) |
|
continue; |
|
list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr, |
|
list[i].size, |
|
DMA_BIDIRECTIONAL); |
|
if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) { |
|
dev_err(&pdev->dev, "Dma mapping failed\n"); |
|
ret = -EIO; |
|
goto sg_cleanup; |
|
} |
|
} |
|
components = buf_count / 4; |
|
sg_ptr = (struct otx2_cpt_sglist_component *)buffer; |
|
for (i = 0; i < components; i++) { |
|
sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size); |
|
sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size); |
|
sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size); |
|
sg_ptr->len3 = cpu_to_be16(list[i * 4 + 3].size); |
|
sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); |
|
sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); |
|
sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); |
|
sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr); |
|
sg_ptr++; |
|
} |
|
components = buf_count % 4; |
|
|
|
switch (components) { |
|
case 3: |
|
sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size); |
|
sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); |
|
fallthrough; |
|
case 2: |
|
sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size); |
|
sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); |
|
fallthrough; |
|
case 1: |
|
sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size); |
|
sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); |
|
break; |
|
default: |
|
break; |
|
} |
|
return ret; |
|
|
|
sg_cleanup: |
|
for (j = 0; j < i; j++) { |
|
if (list[j].dma_addr) { |
|
dma_unmap_single(&pdev->dev, list[j].dma_addr, |
|
list[j].size, DMA_BIDIRECTIONAL); |
|
} |
|
|
|
list[j].dma_addr = 0; |
|
} |
|
return ret; |
|
} |
|
|
|
static inline struct otx2_cpt_inst_info *info_create(struct pci_dev *pdev, |
|
struct otx2_cpt_req_info *req, |
|
gfp_t gfp) |
|
{ |
|
int align = OTX2_CPT_DMA_MINALIGN; |
|
struct otx2_cpt_inst_info *info; |
|
u32 dlen, align_dlen, info_len; |
|
u16 g_sz_bytes, s_sz_bytes; |
|
u32 total_mem_len; |
|
|
|
if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT || |
|
req->out_cnt > OTX2_CPT_MAX_SG_OUT_CNT)) { |
|
dev_err(&pdev->dev, "Error too many sg components\n"); |
|
return NULL; |
|
} |
|
|
|
g_sz_bytes = ((req->in_cnt + 3) / 4) * |
|
sizeof(struct otx2_cpt_sglist_component); |
|
s_sz_bytes = ((req->out_cnt + 3) / 4) * |
|
sizeof(struct otx2_cpt_sglist_component); |
|
|
|
dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; |
|
align_dlen = ALIGN(dlen, align); |
|
info_len = ALIGN(sizeof(*info), align); |
|
total_mem_len = align_dlen + info_len + sizeof(union otx2_cpt_res_s); |
|
|
|
info = kzalloc(total_mem_len, gfp); |
|
if (unlikely(!info)) |
|
return NULL; |
|
|
|
info->dlen = dlen; |
|
info->in_buffer = (u8 *)info + info_len; |
|
|
|
((u16 *)info->in_buffer)[0] = req->out_cnt; |
|
((u16 *)info->in_buffer)[1] = req->in_cnt; |
|
((u16 *)info->in_buffer)[2] = 0; |
|
((u16 *)info->in_buffer)[3] = 0; |
|
cpu_to_be64s((u64 *)info->in_buffer); |
|
|
|
/* Setup gather (input) components */ |
|
if (setup_sgio_components(pdev, req->in, req->in_cnt, |
|
&info->in_buffer[8])) { |
|
dev_err(&pdev->dev, "Failed to setup gather list\n"); |
|
goto destroy_info; |
|
} |
|
|
|
if (setup_sgio_components(pdev, req->out, req->out_cnt, |
|
&info->in_buffer[8 + g_sz_bytes])) { |
|
dev_err(&pdev->dev, "Failed to setup scatter list\n"); |
|
goto destroy_info; |
|
} |
|
|
|
info->dma_len = total_mem_len - info_len; |
|
info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer, |
|
info->dma_len, DMA_BIDIRECTIONAL); |
|
if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) { |
|
dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n"); |
|
goto destroy_info; |
|
} |
|
/* |
|
* Get buffer for union otx2_cpt_res_s response |
|
* structure and its physical address |
|
*/ |
|
info->completion_addr = info->in_buffer + align_dlen; |
|
info->comp_baddr = info->dptr_baddr + align_dlen; |
|
|
|
return info; |
|
|
|
destroy_info: |
|
otx2_cpt_info_destroy(pdev, info); |
|
return NULL; |
|
} |
|
|
|
static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, |
|
struct otx2_cpt_pending_queue *pqueue, |
|
struct otx2_cptlf_info *lf) |
|
{ |
|
struct otx2_cptvf_request *cpt_req = &req->req; |
|
struct otx2_cpt_pending_entry *pentry = NULL; |
|
union otx2_cpt_ctrl_info *ctrl = &req->ctrl; |
|
struct otx2_cpt_inst_info *info = NULL; |
|
union otx2_cpt_res_s *result = NULL; |
|
struct otx2_cpt_iq_command iq_cmd; |
|
union otx2_cpt_inst_s cptinst; |
|
int retry, ret = 0; |
|
u8 resume_sender; |
|
gfp_t gfp; |
|
|
|
gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : |
|
GFP_ATOMIC; |
|
if (unlikely(!otx2_cptlf_started(lf->lfs))) |
|
return -ENODEV; |
|
|
|
info = info_create(pdev, req, gfp); |
|
if (unlikely(!info)) { |
|
dev_err(&pdev->dev, "Setting up cpt inst info failed"); |
|
return -ENOMEM; |
|
} |
|
cpt_req->dlen = info->dlen; |
|
|
|
result = info->completion_addr; |
|
result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT; |
|
|
|
spin_lock_bh(&pqueue->lock); |
|
pentry = get_free_pending_entry(pqueue, pqueue->qlen); |
|
retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP; |
|
while (unlikely(!pentry) && retry--) { |
|
spin_unlock_bh(&pqueue->lock); |
|
udelay(CPT_PENTRY_STEP); |
|
spin_lock_bh(&pqueue->lock); |
|
pentry = get_free_pending_entry(pqueue, pqueue->qlen); |
|
} |
|
|
|
if (unlikely(!pentry)) { |
|
ret = -ENOSPC; |
|
goto destroy_info; |
|
} |
|
|
|
/* |
|
* Check if we are close to filling in entire pending queue, |
|
* if so then tell the sender to stop/sleep by returning -EBUSY |
|
* We do it only for context which can sleep (GFP_KERNEL) |
|
*/ |
|
if (gfp == GFP_KERNEL && |
|
pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) { |
|
pentry->resume_sender = true; |
|
} else |
|
pentry->resume_sender = false; |
|
resume_sender = pentry->resume_sender; |
|
pqueue->pending_count++; |
|
|
|
pentry->completion_addr = info->completion_addr; |
|
pentry->info = info; |
|
pentry->callback = req->callback; |
|
pentry->areq = req->areq; |
|
pentry->busy = true; |
|
info->pentry = pentry; |
|
info->time_in = jiffies; |
|
info->req = req; |
|
|
|
/* Fill in the command */ |
|
iq_cmd.cmd.u = 0; |
|
iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags); |
|
iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1); |
|
iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2); |
|
iq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen); |
|
|
|
/* 64-bit swap for microcode data reads, not needed for addresses*/ |
|
cpu_to_be64s(&iq_cmd.cmd.u); |
|
iq_cmd.dptr = info->dptr_baddr; |
|
iq_cmd.rptr = 0; |
|
iq_cmd.cptr.u = 0; |
|
iq_cmd.cptr.s.grp = ctrl->s.grp; |
|
|
|
/* Fill in the CPT_INST_S type command for HW interpretation */ |
|
otx2_cpt_fill_inst(&cptinst, &iq_cmd, info->comp_baddr); |
|
|
|
/* Print debug info if enabled */ |
|
otx2_cpt_dump_sg_list(pdev, req); |
|
pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX2_CPT_INST_SIZE); |
|
print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX2_CPT_INST_SIZE, false); |
|
pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen); |
|
print_hex_dump_debug("", 0, 16, 1, info->in_buffer, |
|
cpt_req->dlen, false); |
|
|
|
/* Send CPT command */ |
|
lf->lfs->ops->send_cmd(&cptinst, 1, lf); |
|
|
|
/* |
|
* We allocate and prepare pending queue entry in critical section |
|
* together with submitting CPT instruction to CPT instruction queue |
|
* to make sure that order of CPT requests is the same in both |
|
* pending and instruction queues |
|
*/ |
|
spin_unlock_bh(&pqueue->lock); |
|
|
|
ret = resume_sender ? -EBUSY : -EINPROGRESS; |
|
return ret; |
|
|
|
destroy_info: |
|
spin_unlock_bh(&pqueue->lock); |
|
otx2_cpt_info_destroy(pdev, info); |
|
return ret; |
|
} |
|
|
|
int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, |
|
int cpu_num) |
|
{ |
|
struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); |
|
struct otx2_cptlfs_info *lfs = &cptvf->lfs; |
|
|
|
return process_request(lfs->pdev, req, &lfs->lf[cpu_num].pqueue, |
|
&lfs->lf[cpu_num]); |
|
} |
|
|
|
static int cpt_process_ccode(struct otx2_cptlfs_info *lfs, |
|
union otx2_cpt_res_s *cpt_status, |
|
struct otx2_cpt_inst_info *info, |
|
u32 *res_code) |
|
{ |
|
u8 uc_ccode = lfs->ops->cpt_get_uc_compcode(cpt_status); |
|
u8 ccode = lfs->ops->cpt_get_compcode(cpt_status); |
|
struct pci_dev *pdev = lfs->pdev; |
|
|
|
switch (ccode) { |
|
case OTX2_CPT_COMP_E_FAULT: |
|
dev_err(&pdev->dev, |
|
"Request failed with DMA fault\n"); |
|
otx2_cpt_dump_sg_list(pdev, info->req); |
|
break; |
|
|
|
case OTX2_CPT_COMP_E_HWERR: |
|
dev_err(&pdev->dev, |
|
"Request failed with hardware error\n"); |
|
otx2_cpt_dump_sg_list(pdev, info->req); |
|
break; |
|
|
|
case OTX2_CPT_COMP_E_INSTERR: |
|
dev_err(&pdev->dev, |
|
"Request failed with instruction error\n"); |
|
otx2_cpt_dump_sg_list(pdev, info->req); |
|
break; |
|
|
|
case OTX2_CPT_COMP_E_NOTDONE: |
|
/* check for timeout */ |
|
if (time_after_eq(jiffies, info->time_in + |
|
CPT_COMMAND_TIMEOUT * HZ)) |
|
dev_warn(&pdev->dev, |
|
"Request timed out 0x%p", info->req); |
|
else if (info->extra_time < CPT_TIME_IN_RESET_COUNT) { |
|
info->time_in = jiffies; |
|
info->extra_time++; |
|
} |
|
return 1; |
|
|
|
case OTX2_CPT_COMP_E_GOOD: |
|
case OTX2_CPT_COMP_E_WARN: |
|
/* |
|
* Check microcode completion code, it is only valid |
|
* when completion code is CPT_COMP_E::GOOD |
|
*/ |
|
if (uc_ccode != OTX2_CPT_UCC_SUCCESS) { |
|
/* |
|
* If requested hmac is truncated and ucode returns |
|
* s/g write length error then we report success |
|
* because ucode writes as many bytes of calculated |
|
* hmac as available in gather buffer and reports |
|
* s/g write length error if number of bytes in gather |
|
* buffer is less than full hmac size. |
|
*/ |
|
if (info->req->is_trunc_hmac && |
|
uc_ccode == OTX2_CPT_UCC_SG_WRITE_LENGTH) { |
|
*res_code = 0; |
|
break; |
|
} |
|
|
|
dev_err(&pdev->dev, |
|
"Request failed with software error code 0x%x\n", |
|
cpt_status->s.uc_compcode); |
|
otx2_cpt_dump_sg_list(pdev, info->req); |
|
break; |
|
} |
|
/* Request has been processed with success */ |
|
*res_code = 0; |
|
break; |
|
|
|
default: |
|
dev_err(&pdev->dev, |
|
"Request returned invalid status %d\n", ccode); |
|
break; |
|
} |
|
return 0; |
|
} |
|
|
|
static inline void process_pending_queue(struct otx2_cptlfs_info *lfs, |
|
struct otx2_cpt_pending_queue *pqueue) |
|
{ |
|
struct otx2_cpt_pending_entry *resume_pentry = NULL; |
|
void (*callback)(int status, void *arg, void *req); |
|
struct otx2_cpt_pending_entry *pentry = NULL; |
|
union otx2_cpt_res_s *cpt_status = NULL; |
|
struct otx2_cpt_inst_info *info = NULL; |
|
struct otx2_cpt_req_info *req = NULL; |
|
struct crypto_async_request *areq; |
|
struct pci_dev *pdev = lfs->pdev; |
|
u32 res_code, resume_index; |
|
|
|
while (1) { |
|
spin_lock_bh(&pqueue->lock); |
|
pentry = &pqueue->head[pqueue->front]; |
|
|
|
if (WARN_ON(!pentry)) { |
|
spin_unlock_bh(&pqueue->lock); |
|
break; |
|
} |
|
|
|
res_code = -EINVAL; |
|
if (unlikely(!pentry->busy)) { |
|
spin_unlock_bh(&pqueue->lock); |
|
break; |
|
} |
|
|
|
if (unlikely(!pentry->callback)) { |
|
dev_err(&pdev->dev, "Callback NULL\n"); |
|
goto process_pentry; |
|
} |
|
|
|
info = pentry->info; |
|
if (unlikely(!info)) { |
|
dev_err(&pdev->dev, "Pending entry post arg NULL\n"); |
|
goto process_pentry; |
|
} |
|
|
|
req = info->req; |
|
if (unlikely(!req)) { |
|
dev_err(&pdev->dev, "Request NULL\n"); |
|
goto process_pentry; |
|
} |
|
|
|
cpt_status = pentry->completion_addr; |
|
if (unlikely(!cpt_status)) { |
|
dev_err(&pdev->dev, "Completion address NULL\n"); |
|
goto process_pentry; |
|
} |
|
|
|
if (cpt_process_ccode(lfs, cpt_status, info, &res_code)) { |
|
spin_unlock_bh(&pqueue->lock); |
|
return; |
|
} |
|
info->pdev = pdev; |
|
|
|
process_pentry: |
|
/* |
|
* Check if we should inform sending side to resume |
|
* We do it CPT_IQ_RESUME_MARGIN elements in advance before |
|
* pending queue becomes empty |
|
*/ |
|
resume_index = modulo_inc(pqueue->front, pqueue->qlen, |
|
CPT_IQ_RESUME_MARGIN); |
|
resume_pentry = &pqueue->head[resume_index]; |
|
if (resume_pentry && |
|
resume_pentry->resume_sender) { |
|
resume_pentry->resume_sender = false; |
|
callback = resume_pentry->callback; |
|
areq = resume_pentry->areq; |
|
|
|
if (callback) { |
|
spin_unlock_bh(&pqueue->lock); |
|
|
|
/* |
|
* EINPROGRESS is an indication for sending |
|
* side that it can resume sending requests |
|
*/ |
|
callback(-EINPROGRESS, areq, info); |
|
spin_lock_bh(&pqueue->lock); |
|
} |
|
} |
|
|
|
callback = pentry->callback; |
|
areq = pentry->areq; |
|
free_pentry(pentry); |
|
|
|
pqueue->pending_count--; |
|
pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1); |
|
spin_unlock_bh(&pqueue->lock); |
|
|
|
/* |
|
* Call callback after current pending entry has been |
|
* processed, we don't do it if the callback pointer is |
|
* invalid. |
|
*/ |
|
if (callback) |
|
callback(res_code, areq, info); |
|
} |
|
} |
|
|
|
void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe) |
|
{ |
|
process_pending_queue(wqe->lfs, |
|
&wqe->lfs->lf[wqe->lf_num].pqueue); |
|
} |
|
|
|
int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev) |
|
{ |
|
struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); |
|
|
|
return cptvf->lfs.kcrypto_eng_grp_num; |
|
}
|
|
|