forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
433 lines
11 KiB
433 lines
11 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* Copyright (C) 2020 Marvell. */ |
|
|
|
#include "otx2_cpt_common.h" |
|
#include "otx2_cptlf.h" |
|
#include "rvu_reg.h" |
|
|
|
#define CPT_TIMER_HOLD 0x03F |
|
#define CPT_COUNT_HOLD 32 |
|
|
|
static void cptlf_do_set_done_time_wait(struct otx2_cptlf_info *lf, |
|
int time_wait) |
|
{ |
|
union otx2_cptx_lf_done_wait done_wait; |
|
|
|
done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, |
|
OTX2_CPT_LF_DONE_WAIT); |
|
done_wait.s.time_wait = time_wait; |
|
otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, |
|
OTX2_CPT_LF_DONE_WAIT, done_wait.u); |
|
} |
|
|
|
static void cptlf_do_set_done_num_wait(struct otx2_cptlf_info *lf, int num_wait) |
|
{ |
|
union otx2_cptx_lf_done_wait done_wait; |
|
|
|
done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, |
|
OTX2_CPT_LF_DONE_WAIT); |
|
done_wait.s.num_wait = num_wait; |
|
otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, |
|
OTX2_CPT_LF_DONE_WAIT, done_wait.u); |
|
} |
|
|
|
static void cptlf_set_done_time_wait(struct otx2_cptlfs_info *lfs, |
|
int time_wait) |
|
{ |
|
int slot; |
|
|
|
for (slot = 0; slot < lfs->lfs_num; slot++) |
|
cptlf_do_set_done_time_wait(&lfs->lf[slot], time_wait); |
|
} |
|
|
|
static void cptlf_set_done_num_wait(struct otx2_cptlfs_info *lfs, int num_wait) |
|
{ |
|
int slot; |
|
|
|
for (slot = 0; slot < lfs->lfs_num; slot++) |
|
cptlf_do_set_done_num_wait(&lfs->lf[slot], num_wait); |
|
} |
|
|
|
static int cptlf_set_pri(struct otx2_cptlf_info *lf, int pri) |
|
{ |
|
struct otx2_cptlfs_info *lfs = lf->lfs; |
|
union otx2_cptx_af_lf_ctrl lf_ctrl; |
|
int ret; |
|
|
|
ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev, |
|
CPT_AF_LFX_CTL(lf->slot), |
|
&lf_ctrl.u, lfs->blkaddr); |
|
if (ret) |
|
return ret; |
|
|
|
lf_ctrl.s.pri = pri ? 1 : 0; |
|
|
|
ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev, |
|
CPT_AF_LFX_CTL(lf->slot), |
|
lf_ctrl.u, lfs->blkaddr); |
|
return ret; |
|
} |
|
|
|
static int cptlf_set_eng_grps_mask(struct otx2_cptlf_info *lf, |
|
int eng_grps_mask) |
|
{ |
|
struct otx2_cptlfs_info *lfs = lf->lfs; |
|
union otx2_cptx_af_lf_ctrl lf_ctrl; |
|
int ret; |
|
|
|
ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev, |
|
CPT_AF_LFX_CTL(lf->slot), |
|
&lf_ctrl.u, lfs->blkaddr); |
|
if (ret) |
|
return ret; |
|
|
|
lf_ctrl.s.grp = eng_grps_mask; |
|
|
|
ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev, |
|
CPT_AF_LFX_CTL(lf->slot), |
|
lf_ctrl.u, lfs->blkaddr); |
|
return ret; |
|
} |
|
|
|
static int cptlf_set_grp_and_pri(struct otx2_cptlfs_info *lfs, |
|
int eng_grp_mask, int pri) |
|
{ |
|
int slot, ret = 0; |
|
|
|
for (slot = 0; slot < lfs->lfs_num; slot++) { |
|
ret = cptlf_set_pri(&lfs->lf[slot], pri); |
|
if (ret) |
|
return ret; |
|
|
|
ret = cptlf_set_eng_grps_mask(&lfs->lf[slot], eng_grp_mask); |
|
if (ret) |
|
return ret; |
|
} |
|
return ret; |
|
} |
|
|
|
static void cptlf_hw_init(struct otx2_cptlfs_info *lfs) |
|
{ |
|
/* Disable instruction queues */ |
|
otx2_cptlf_disable_iqueues(lfs); |
|
|
|
/* Set instruction queues base addresses */ |
|
otx2_cptlf_set_iqueues_base_addr(lfs); |
|
|
|
/* Set instruction queues sizes */ |
|
otx2_cptlf_set_iqueues_size(lfs); |
|
|
|
/* Set done interrupts time wait */ |
|
cptlf_set_done_time_wait(lfs, CPT_TIMER_HOLD); |
|
|
|
/* Set done interrupts num wait */ |
|
cptlf_set_done_num_wait(lfs, CPT_COUNT_HOLD); |
|
|
|
/* Enable instruction queues */ |
|
otx2_cptlf_enable_iqueues(lfs); |
|
} |
|
|
|
static void cptlf_hw_cleanup(struct otx2_cptlfs_info *lfs) |
|
{ |
|
/* Disable instruction queues */ |
|
otx2_cptlf_disable_iqueues(lfs); |
|
} |
|
|
|
static void cptlf_set_misc_intrs(struct otx2_cptlfs_info *lfs, u8 enable) |
|
{ |
|
union otx2_cptx_lf_misc_int_ena_w1s irq_misc = { .u = 0x0 }; |
|
u64 reg = enable ? OTX2_CPT_LF_MISC_INT_ENA_W1S : |
|
OTX2_CPT_LF_MISC_INT_ENA_W1C; |
|
int slot; |
|
|
|
irq_misc.s.fault = 0x1; |
|
irq_misc.s.hwerr = 0x1; |
|
irq_misc.s.irde = 0x1; |
|
irq_misc.s.nqerr = 0x1; |
|
irq_misc.s.nwrp = 0x1; |
|
|
|
for (slot = 0; slot < lfs->lfs_num; slot++) |
|
otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, reg, |
|
irq_misc.u); |
|
} |
|
|
|
static void cptlf_enable_intrs(struct otx2_cptlfs_info *lfs) |
|
{ |
|
int slot; |
|
|
|
/* Enable done interrupts */ |
|
for (slot = 0; slot < lfs->lfs_num; slot++) |
|
otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, |
|
OTX2_CPT_LF_DONE_INT_ENA_W1S, 0x1); |
|
/* Enable Misc interrupts */ |
|
cptlf_set_misc_intrs(lfs, true); |
|
} |
|
|
|
static void cptlf_disable_intrs(struct otx2_cptlfs_info *lfs) |
|
{ |
|
int slot; |
|
|
|
for (slot = 0; slot < lfs->lfs_num; slot++) |
|
otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, |
|
OTX2_CPT_LF_DONE_INT_ENA_W1C, 0x1); |
|
cptlf_set_misc_intrs(lfs, false); |
|
} |
|
|
|
static inline int cptlf_read_done_cnt(struct otx2_cptlf_info *lf) |
|
{ |
|
union otx2_cptx_lf_done irq_cnt; |
|
|
|
irq_cnt.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, |
|
OTX2_CPT_LF_DONE); |
|
return irq_cnt.s.done; |
|
} |
|
|
|
static irqreturn_t cptlf_misc_intr_handler(int __always_unused irq, void *arg) |
|
{ |
|
union otx2_cptx_lf_misc_int irq_misc, irq_misc_ack; |
|
struct otx2_cptlf_info *lf = arg; |
|
struct device *dev; |
|
|
|
dev = &lf->lfs->pdev->dev; |
|
irq_misc.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, |
|
OTX2_CPT_LF_MISC_INT); |
|
irq_misc_ack.u = 0x0; |
|
|
|
if (irq_misc.s.fault) { |
|
dev_err(dev, "Memory error detected while executing CPT_INST_S, LF %d.\n", |
|
lf->slot); |
|
irq_misc_ack.s.fault = 0x1; |
|
|
|
} else if (irq_misc.s.hwerr) { |
|
dev_err(dev, "HW error from an engine executing CPT_INST_S, LF %d.", |
|
lf->slot); |
|
irq_misc_ack.s.hwerr = 0x1; |
|
|
|
} else if (irq_misc.s.nwrp) { |
|
dev_err(dev, "SMMU fault while writing CPT_RES_S to CPT_INST_S[RES_ADDR], LF %d.\n", |
|
lf->slot); |
|
irq_misc_ack.s.nwrp = 0x1; |
|
|
|
} else if (irq_misc.s.irde) { |
|
dev_err(dev, "Memory error when accessing instruction memory queue CPT_LF_Q_BASE[ADDR].\n"); |
|
irq_misc_ack.s.irde = 0x1; |
|
|
|
} else if (irq_misc.s.nqerr) { |
|
dev_err(dev, "Error enqueuing an instruction received at CPT_LF_NQ.\n"); |
|
irq_misc_ack.s.nqerr = 0x1; |
|
|
|
} else { |
|
dev_err(dev, "Unhandled interrupt in CPT LF %d\n", lf->slot); |
|
return IRQ_NONE; |
|
} |
|
|
|
/* Acknowledge interrupts */ |
|
otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, |
|
OTX2_CPT_LF_MISC_INT, irq_misc_ack.u); |
|
|
|
return IRQ_HANDLED; |
|
} |
|
|
|
static irqreturn_t cptlf_done_intr_handler(int irq, void *arg) |
|
{ |
|
union otx2_cptx_lf_done_wait done_wait; |
|
struct otx2_cptlf_info *lf = arg; |
|
int irq_cnt; |
|
|
|
/* Read the number of completed requests */ |
|
irq_cnt = cptlf_read_done_cnt(lf); |
|
if (irq_cnt) { |
|
done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, |
|
lf->slot, OTX2_CPT_LF_DONE_WAIT); |
|
/* Acknowledge the number of completed requests */ |
|
otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, |
|
OTX2_CPT_LF_DONE_ACK, irq_cnt); |
|
|
|
otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, |
|
OTX2_CPT_LF_DONE_WAIT, done_wait.u); |
|
if (unlikely(!lf->wqe)) { |
|
dev_err(&lf->lfs->pdev->dev, "No work for LF %d\n", |
|
lf->slot); |
|
return IRQ_NONE; |
|
} |
|
|
|
/* Schedule processing of completed requests */ |
|
tasklet_hi_schedule(&lf->wqe->work); |
|
} |
|
return IRQ_HANDLED; |
|
} |
|
|
|
void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs) |
|
{ |
|
int i, offs, vector; |
|
|
|
for (i = 0; i < lfs->lfs_num; i++) { |
|
for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) { |
|
if (!lfs->lf[i].is_irq_reg[offs]) |
|
continue; |
|
|
|
vector = pci_irq_vector(lfs->pdev, |
|
lfs->lf[i].msix_offset + offs); |
|
free_irq(vector, &lfs->lf[i]); |
|
lfs->lf[i].is_irq_reg[offs] = false; |
|
} |
|
} |
|
cptlf_disable_intrs(lfs); |
|
} |
|
|
|
static int cptlf_do_register_interrrupts(struct otx2_cptlfs_info *lfs, |
|
int lf_num, int irq_offset, |
|
irq_handler_t handler) |
|
{ |
|
int ret, vector; |
|
|
|
vector = pci_irq_vector(lfs->pdev, lfs->lf[lf_num].msix_offset + |
|
irq_offset); |
|
ret = request_irq(vector, handler, 0, |
|
lfs->lf[lf_num].irq_name[irq_offset], |
|
&lfs->lf[lf_num]); |
|
if (ret) |
|
return ret; |
|
|
|
lfs->lf[lf_num].is_irq_reg[irq_offset] = true; |
|
|
|
return ret; |
|
} |
|
|
|
int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs) |
|
{ |
|
int irq_offs, ret, i; |
|
|
|
for (i = 0; i < lfs->lfs_num; i++) { |
|
irq_offs = OTX2_CPT_LF_INT_VEC_E_MISC; |
|
snprintf(lfs->lf[i].irq_name[irq_offs], 32, "CPTLF Misc%d", i); |
|
ret = cptlf_do_register_interrrupts(lfs, i, irq_offs, |
|
cptlf_misc_intr_handler); |
|
if (ret) |
|
goto free_irq; |
|
|
|
irq_offs = OTX2_CPT_LF_INT_VEC_E_DONE; |
|
snprintf(lfs->lf[i].irq_name[irq_offs], 32, "OTX2_CPTLF Done%d", |
|
i); |
|
ret = cptlf_do_register_interrrupts(lfs, i, irq_offs, |
|
cptlf_done_intr_handler); |
|
if (ret) |
|
goto free_irq; |
|
} |
|
cptlf_enable_intrs(lfs); |
|
return 0; |
|
|
|
free_irq: |
|
otx2_cptlf_unregister_interrupts(lfs); |
|
return ret; |
|
} |
|
|
|
void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs) |
|
{ |
|
int slot, offs; |
|
|
|
for (slot = 0; slot < lfs->lfs_num; slot++) { |
|
for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) |
|
irq_set_affinity_hint(pci_irq_vector(lfs->pdev, |
|
lfs->lf[slot].msix_offset + |
|
offs), NULL); |
|
free_cpumask_var(lfs->lf[slot].affinity_mask); |
|
} |
|
} |
|
|
|
int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs) |
|
{ |
|
struct otx2_cptlf_info *lf = lfs->lf; |
|
int slot, offs, ret; |
|
|
|
for (slot = 0; slot < lfs->lfs_num; slot++) { |
|
if (!zalloc_cpumask_var(&lf[slot].affinity_mask, GFP_KERNEL)) { |
|
dev_err(&lfs->pdev->dev, |
|
"cpumask allocation failed for LF %d", slot); |
|
ret = -ENOMEM; |
|
goto free_affinity_mask; |
|
} |
|
|
|
cpumask_set_cpu(cpumask_local_spread(slot, |
|
dev_to_node(&lfs->pdev->dev)), |
|
lf[slot].affinity_mask); |
|
|
|
for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) { |
|
ret = irq_set_affinity_hint(pci_irq_vector(lfs->pdev, |
|
lf[slot].msix_offset + offs), |
|
lf[slot].affinity_mask); |
|
if (ret) |
|
goto free_affinity_mask; |
|
} |
|
} |
|
return 0; |
|
|
|
free_affinity_mask: |
|
otx2_cptlf_free_irqs_affinity(lfs); |
|
return ret; |
|
} |
|
|
|
int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_mask, int pri, |
|
int lfs_num) |
|
{ |
|
int slot, ret; |
|
|
|
if (!lfs->pdev || !lfs->reg_base) |
|
return -EINVAL; |
|
|
|
lfs->lfs_num = lfs_num; |
|
for (slot = 0; slot < lfs->lfs_num; slot++) { |
|
lfs->lf[slot].lfs = lfs; |
|
lfs->lf[slot].slot = slot; |
|
if (lfs->lmt_base) |
|
lfs->lf[slot].lmtline = lfs->lmt_base + |
|
(slot * LMTLINE_SIZE); |
|
else |
|
lfs->lf[slot].lmtline = lfs->reg_base + |
|
OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_LMT, slot, |
|
OTX2_CPT_LMT_LF_LMTLINEX(0)); |
|
|
|
lfs->lf[slot].ioreg = lfs->reg_base + |
|
OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_CPT0, slot, |
|
OTX2_CPT_LF_NQX(0)); |
|
} |
|
/* Send request to attach LFs */ |
|
ret = otx2_cpt_attach_rscrs_msg(lfs); |
|
if (ret) |
|
goto clear_lfs_num; |
|
|
|
ret = otx2_cpt_alloc_instruction_queues(lfs); |
|
if (ret) { |
|
dev_err(&lfs->pdev->dev, |
|
"Allocating instruction queues failed\n"); |
|
goto detach_rsrcs; |
|
} |
|
cptlf_hw_init(lfs); |
|
/* |
|
* Allow each LF to execute requests destined to any of 8 engine |
|
* groups and set queue priority of each LF to high |
|
*/ |
|
ret = cptlf_set_grp_and_pri(lfs, eng_grp_mask, pri); |
|
if (ret) |
|
goto free_iq; |
|
|
|
return 0; |
|
|
|
free_iq: |
|
otx2_cpt_free_instruction_queues(lfs); |
|
cptlf_hw_cleanup(lfs); |
|
detach_rsrcs: |
|
otx2_cpt_detach_rsrcs_msg(lfs); |
|
clear_lfs_num: |
|
lfs->lfs_num = 0; |
|
return ret; |
|
} |
|
|
|
void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs) |
|
{ |
|
lfs->lfs_num = 0; |
|
/* Cleanup LFs hardware side */ |
|
cptlf_hw_cleanup(lfs); |
|
/* Send request to detach LFs */ |
|
otx2_cpt_detach_rsrcs_msg(lfs); |
|
}
|
|
|