mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
970 lines
26 KiB
970 lines
26 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* |
|
* Huawei HiNIC PCI Express Linux driver |
|
* Copyright(c) 2017 Huawei Technologies Co., Ltd |
|
*/ |
|
|
|
#include <linux/kernel.h> |
|
#include <linux/types.h> |
|
#include <linux/errno.h> |
|
#include <linux/pci.h> |
|
#include <linux/device.h> |
|
#include <linux/slab.h> |
|
#include <linux/vmalloc.h> |
|
#include <linux/spinlock.h> |
|
#include <linux/sizes.h> |
|
#include <linux/atomic.h> |
|
#include <linux/log2.h> |
|
#include <linux/io.h> |
|
#include <linux/completion.h> |
|
#include <linux/err.h> |
|
#include <asm/byteorder.h> |
|
#include <asm/barrier.h> |
|
|
|
#include "hinic_common.h" |
|
#include "hinic_hw_if.h" |
|
#include "hinic_hw_eqs.h" |
|
#include "hinic_hw_mgmt.h" |
|
#include "hinic_hw_wqe.h" |
|
#include "hinic_hw_wq.h" |
|
#include "hinic_hw_cmdq.h" |
|
#include "hinic_hw_io.h" |
|
#include "hinic_hw_dev.h" |
|
|
|
#define CMDQ_CEQE_TYPE_SHIFT 0 |
|
|
|
#define CMDQ_CEQE_TYPE_MASK 0x7 |
|
|
|
#define CMDQ_CEQE_GET(val, member) \ |
|
(((val) >> CMDQ_CEQE_##member##_SHIFT) \ |
|
& CMDQ_CEQE_##member##_MASK) |
|
|
|
#define CMDQ_WQE_ERRCODE_VAL_SHIFT 20 |
|
|
|
#define CMDQ_WQE_ERRCODE_VAL_MASK 0xF |
|
|
|
#define CMDQ_WQE_ERRCODE_GET(val, member) \ |
|
(((val) >> CMDQ_WQE_ERRCODE_##member##_SHIFT) \ |
|
& CMDQ_WQE_ERRCODE_##member##_MASK) |
|
|
|
#define CMDQ_DB_PI_OFF(pi) (((u16)LOWER_8_BITS(pi)) << 3) |
|
|
|
#define CMDQ_DB_ADDR(db_base, pi) ((db_base) + CMDQ_DB_PI_OFF(pi)) |
|
|
|
#define CMDQ_WQE_HEADER(wqe) ((struct hinic_cmdq_header *)(wqe)) |
|
|
|
#define CMDQ_WQE_COMPLETED(ctrl_info) \ |
|
HINIC_CMDQ_CTRL_GET(ctrl_info, HW_BUSY_BIT) |
|
|
|
#define FIRST_DATA_TO_WRITE_LAST sizeof(u64) |
|
|
|
#define CMDQ_DB_OFF SZ_2K |
|
|
|
#define CMDQ_WQEBB_SIZE 64 |
|
#define CMDQ_WQE_SIZE 64 |
|
#define CMDQ_DEPTH SZ_4K |
|
|
|
#define CMDQ_WQ_PAGE_SIZE SZ_256K |
|
|
|
#define WQE_LCMD_SIZE 64 |
|
#define WQE_SCMD_SIZE 64 |
|
|
|
#define COMPLETE_LEN 3 |
|
|
|
#define CMDQ_TIMEOUT 1000 |
|
|
|
#define CMDQ_PFN(addr, page_size) ((addr) >> (ilog2(page_size))) |
|
|
|
#define cmdq_to_cmdqs(cmdq) container_of((cmdq) - (cmdq)->cmdq_type, \ |
|
struct hinic_cmdqs, cmdq[0]) |
|
|
|
#define cmdqs_to_func_to_io(cmdqs) container_of(cmdqs, \ |
|
struct hinic_func_to_io, \ |
|
cmdqs) |
|
|
|
enum cmdq_wqe_type { |
|
WQE_LCMD_TYPE = 0, |
|
WQE_SCMD_TYPE = 1, |
|
}; |
|
|
|
enum completion_format { |
|
COMPLETE_DIRECT = 0, |
|
COMPLETE_SGE = 1, |
|
}; |
|
|
|
enum data_format { |
|
DATA_SGE = 0, |
|
DATA_DIRECT = 1, |
|
}; |
|
|
|
enum bufdesc_len { |
|
BUFDESC_LCMD_LEN = 2, /* 16 bytes - 2(8 byte unit) */ |
|
BUFDESC_SCMD_LEN = 3, /* 24 bytes - 3(8 byte unit) */ |
|
}; |
|
|
|
enum ctrl_sect_len { |
|
CTRL_SECT_LEN = 1, /* 4 bytes (ctrl) - 1(8 byte unit) */ |
|
CTRL_DIRECT_SECT_LEN = 2, /* 12 bytes (ctrl + rsvd) - 2(8 byte unit) */ |
|
}; |
|
|
|
enum cmdq_scmd_type { |
|
CMDQ_SET_ARM_CMD = 2, |
|
}; |
|
|
|
enum cmdq_cmd_type { |
|
CMDQ_CMD_SYNC_DIRECT_RESP = 0, |
|
CMDQ_CMD_SYNC_SGE_RESP = 1, |
|
}; |
|
|
|
enum completion_request { |
|
NO_CEQ = 0, |
|
CEQ_SET = 1, |
|
}; |
|
|
|
/** |
|
* hinic_alloc_cmdq_buf - alloc buffer for sending command |
|
* @cmdqs: the cmdqs |
|
* @cmdq_buf: the buffer returned in this struct |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
int hinic_alloc_cmdq_buf(struct hinic_cmdqs *cmdqs, |
|
struct hinic_cmdq_buf *cmdq_buf) |
|
{ |
|
struct hinic_hwif *hwif = cmdqs->hwif; |
|
struct pci_dev *pdev = hwif->pdev; |
|
|
|
cmdq_buf->buf = dma_pool_alloc(cmdqs->cmdq_buf_pool, GFP_KERNEL, |
|
&cmdq_buf->dma_addr); |
|
if (!cmdq_buf->buf) { |
|
dev_err(&pdev->dev, "Failed to allocate cmd from the pool\n"); |
|
return -ENOMEM; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
/** |
|
* hinic_free_cmdq_buf - free buffer |
|
* @cmdqs: the cmdqs |
|
* @cmdq_buf: the buffer to free that is in this struct |
|
**/ |
|
void hinic_free_cmdq_buf(struct hinic_cmdqs *cmdqs, |
|
struct hinic_cmdq_buf *cmdq_buf) |
|
{ |
|
dma_pool_free(cmdqs->cmdq_buf_pool, cmdq_buf->buf, cmdq_buf->dma_addr); |
|
} |
|
|
|
static unsigned int cmdq_wqe_size_from_bdlen(enum bufdesc_len len) |
|
{ |
|
unsigned int wqe_size = 0; |
|
|
|
switch (len) { |
|
case BUFDESC_LCMD_LEN: |
|
wqe_size = WQE_LCMD_SIZE; |
|
break; |
|
case BUFDESC_SCMD_LEN: |
|
wqe_size = WQE_SCMD_SIZE; |
|
break; |
|
} |
|
|
|
return wqe_size; |
|
} |
|
|
|
static void cmdq_set_sge_completion(struct hinic_cmdq_completion *completion, |
|
struct hinic_cmdq_buf *buf_out) |
|
{ |
|
struct hinic_sge_resp *sge_resp = &completion->sge_resp; |
|
|
|
hinic_set_sge(&sge_resp->sge, buf_out->dma_addr, buf_out->size); |
|
} |
|
|
|
static void cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe *wqe, int wrapped, |
|
enum hinic_cmd_ack_type ack_type, |
|
enum hinic_mod_type mod, u8 cmd, u16 prod_idx, |
|
enum completion_format complete_format, |
|
enum data_format data_format, |
|
enum bufdesc_len buf_len) |
|
{ |
|
struct hinic_cmdq_wqe_lcmd *wqe_lcmd; |
|
struct hinic_cmdq_wqe_scmd *wqe_scmd; |
|
enum ctrl_sect_len ctrl_len; |
|
struct hinic_ctrl *ctrl; |
|
u32 saved_data; |
|
|
|
if (data_format == DATA_SGE) { |
|
wqe_lcmd = &wqe->wqe_lcmd; |
|
|
|
wqe_lcmd->status.status_info = 0; |
|
ctrl = &wqe_lcmd->ctrl; |
|
ctrl_len = CTRL_SECT_LEN; |
|
} else { |
|
wqe_scmd = &wqe->direct_wqe.wqe_scmd; |
|
|
|
wqe_scmd->status.status_info = 0; |
|
ctrl = &wqe_scmd->ctrl; |
|
ctrl_len = CTRL_DIRECT_SECT_LEN; |
|
} |
|
|
|
ctrl->ctrl_info = HINIC_CMDQ_CTRL_SET(prod_idx, PI) | |
|
HINIC_CMDQ_CTRL_SET(cmd, CMD) | |
|
HINIC_CMDQ_CTRL_SET(mod, MOD) | |
|
HINIC_CMDQ_CTRL_SET(ack_type, ACK_TYPE); |
|
|
|
CMDQ_WQE_HEADER(wqe)->header_info = |
|
HINIC_CMDQ_WQE_HEADER_SET(buf_len, BUFDESC_LEN) | |
|
HINIC_CMDQ_WQE_HEADER_SET(complete_format, COMPLETE_FMT) | |
|
HINIC_CMDQ_WQE_HEADER_SET(data_format, DATA_FMT) | |
|
HINIC_CMDQ_WQE_HEADER_SET(CEQ_SET, COMPLETE_REQ) | |
|
HINIC_CMDQ_WQE_HEADER_SET(COMPLETE_LEN, COMPLETE_SECT_LEN) | |
|
HINIC_CMDQ_WQE_HEADER_SET(ctrl_len, CTRL_LEN) | |
|
HINIC_CMDQ_WQE_HEADER_SET(wrapped, TOGGLED_WRAPPED); |
|
|
|
saved_data = CMDQ_WQE_HEADER(wqe)->saved_data; |
|
saved_data = HINIC_SAVED_DATA_CLEAR(saved_data, ARM); |
|
|
|
if ((cmd == CMDQ_SET_ARM_CMD) && (mod == HINIC_MOD_COMM)) |
|
CMDQ_WQE_HEADER(wqe)->saved_data |= |
|
HINIC_SAVED_DATA_SET(1, ARM); |
|
else |
|
CMDQ_WQE_HEADER(wqe)->saved_data = saved_data; |
|
} |
|
|
|
static void cmdq_set_lcmd_bufdesc(struct hinic_cmdq_wqe_lcmd *wqe_lcmd, |
|
struct hinic_cmdq_buf *buf_in) |
|
{ |
|
hinic_set_sge(&wqe_lcmd->buf_desc.sge, buf_in->dma_addr, buf_in->size); |
|
} |
|
|
|
static void cmdq_set_direct_wqe_data(struct hinic_cmdq_direct_wqe *wqe, |
|
void *buf_in, u32 in_size) |
|
{ |
|
struct hinic_cmdq_wqe_scmd *wqe_scmd = &wqe->wqe_scmd; |
|
|
|
wqe_scmd->buf_desc.buf_len = in_size; |
|
memcpy(wqe_scmd->buf_desc.data, buf_in, in_size); |
|
} |
|
|
|
static void cmdq_set_lcmd_wqe(struct hinic_cmdq_wqe *wqe, |
|
enum cmdq_cmd_type cmd_type, |
|
struct hinic_cmdq_buf *buf_in, |
|
struct hinic_cmdq_buf *buf_out, int wrapped, |
|
enum hinic_cmd_ack_type ack_type, |
|
enum hinic_mod_type mod, u8 cmd, u16 prod_idx) |
|
{ |
|
struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd; |
|
enum completion_format complete_format; |
|
|
|
switch (cmd_type) { |
|
case CMDQ_CMD_SYNC_SGE_RESP: |
|
complete_format = COMPLETE_SGE; |
|
cmdq_set_sge_completion(&wqe_lcmd->completion, buf_out); |
|
break; |
|
case CMDQ_CMD_SYNC_DIRECT_RESP: |
|
complete_format = COMPLETE_DIRECT; |
|
wqe_lcmd->completion.direct_resp = 0; |
|
break; |
|
} |
|
|
|
cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, |
|
prod_idx, complete_format, DATA_SGE, |
|
BUFDESC_LCMD_LEN); |
|
|
|
cmdq_set_lcmd_bufdesc(wqe_lcmd, buf_in); |
|
} |
|
|
|
static void cmdq_set_direct_wqe(struct hinic_cmdq_wqe *wqe, |
|
enum cmdq_cmd_type cmd_type, |
|
void *buf_in, u16 in_size, |
|
struct hinic_cmdq_buf *buf_out, int wrapped, |
|
enum hinic_cmd_ack_type ack_type, |
|
enum hinic_mod_type mod, u8 cmd, u16 prod_idx) |
|
{ |
|
struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; |
|
enum completion_format complete_format; |
|
struct hinic_cmdq_wqe_scmd *wqe_scmd; |
|
|
|
wqe_scmd = &direct_wqe->wqe_scmd; |
|
|
|
switch (cmd_type) { |
|
case CMDQ_CMD_SYNC_SGE_RESP: |
|
complete_format = COMPLETE_SGE; |
|
cmdq_set_sge_completion(&wqe_scmd->completion, buf_out); |
|
break; |
|
case CMDQ_CMD_SYNC_DIRECT_RESP: |
|
complete_format = COMPLETE_DIRECT; |
|
wqe_scmd->completion.direct_resp = 0; |
|
break; |
|
} |
|
|
|
cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, prod_idx, |
|
complete_format, DATA_DIRECT, BUFDESC_SCMD_LEN); |
|
|
|
cmdq_set_direct_wqe_data(direct_wqe, buf_in, in_size); |
|
} |
|
|
|
static void cmdq_wqe_fill(void *dst, void *src) |
|
{ |
|
memcpy(dst + FIRST_DATA_TO_WRITE_LAST, src + FIRST_DATA_TO_WRITE_LAST, |
|
CMDQ_WQE_SIZE - FIRST_DATA_TO_WRITE_LAST); |
|
|
|
wmb(); /* The first 8 bytes should be written last */ |
|
|
|
*(u64 *)dst = *(u64 *)src; |
|
} |
|
|
|
static void cmdq_fill_db(u32 *db_info, |
|
enum hinic_cmdq_type cmdq_type, u16 prod_idx) |
|
{ |
|
*db_info = HINIC_CMDQ_DB_INFO_SET(UPPER_8_BITS(prod_idx), HI_PROD_IDX) | |
|
HINIC_CMDQ_DB_INFO_SET(HINIC_CTRL_PATH, PATH) | |
|
HINIC_CMDQ_DB_INFO_SET(cmdq_type, CMDQ_TYPE) | |
|
HINIC_CMDQ_DB_INFO_SET(HINIC_DB_CMDQ_TYPE, DB_TYPE); |
|
} |
|
|
|
static void cmdq_set_db(struct hinic_cmdq *cmdq, |
|
enum hinic_cmdq_type cmdq_type, u16 prod_idx) |
|
{ |
|
u32 db_info; |
|
|
|
cmdq_fill_db(&db_info, cmdq_type, prod_idx); |
|
|
|
/* The data that is written to HW should be in Big Endian Format */ |
|
db_info = cpu_to_be32(db_info); |
|
|
|
wmb(); /* write all before the doorbell */ |
|
|
|
writel(db_info, CMDQ_DB_ADDR(cmdq->db_base, prod_idx)); |
|
} |
|
|
|
static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq, |
|
enum hinic_mod_type mod, u8 cmd, |
|
struct hinic_cmdq_buf *buf_in, |
|
u64 *resp) |
|
{ |
|
struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe; |
|
u16 curr_prod_idx, next_prod_idx; |
|
int errcode, wrapped, num_wqebbs; |
|
struct hinic_wq *wq = cmdq->wq; |
|
struct hinic_hw_wqe *hw_wqe; |
|
struct completion done; |
|
|
|
/* Keep doorbell index correct. bh - for tasklet(ceq). */ |
|
spin_lock_bh(&cmdq->cmdq_lock); |
|
|
|
/* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/ |
|
hw_wqe = hinic_get_wqe(wq, WQE_LCMD_SIZE, &curr_prod_idx); |
|
if (IS_ERR(hw_wqe)) { |
|
spin_unlock_bh(&cmdq->cmdq_lock); |
|
return -EBUSY; |
|
} |
|
|
|
curr_cmdq_wqe = &hw_wqe->cmdq_wqe; |
|
|
|
wrapped = cmdq->wrapped; |
|
|
|
num_wqebbs = ALIGN(WQE_LCMD_SIZE, wq->wqebb_size) / wq->wqebb_size; |
|
next_prod_idx = curr_prod_idx + num_wqebbs; |
|
if (next_prod_idx >= wq->q_depth) { |
|
cmdq->wrapped = !cmdq->wrapped; |
|
next_prod_idx -= wq->q_depth; |
|
} |
|
|
|
cmdq->errcode[curr_prod_idx] = &errcode; |
|
|
|
init_completion(&done); |
|
cmdq->done[curr_prod_idx] = &done; |
|
|
|
cmdq_set_lcmd_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in, NULL, |
|
wrapped, HINIC_CMD_ACK_TYPE_CMDQ, mod, cmd, |
|
curr_prod_idx); |
|
|
|
/* The data that is written to HW should be in Big Endian Format */ |
|
hinic_cpu_to_be32(&cmdq_wqe, WQE_LCMD_SIZE); |
|
|
|
/* CMDQ WQE is not shadow, therefore wqe will be written to wq */ |
|
cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe); |
|
|
|
cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx); |
|
|
|
spin_unlock_bh(&cmdq->cmdq_lock); |
|
|
|
if (!wait_for_completion_timeout(&done, |
|
msecs_to_jiffies(CMDQ_TIMEOUT))) { |
|
spin_lock_bh(&cmdq->cmdq_lock); |
|
|
|
if (cmdq->errcode[curr_prod_idx] == &errcode) |
|
cmdq->errcode[curr_prod_idx] = NULL; |
|
|
|
if (cmdq->done[curr_prod_idx] == &done) |
|
cmdq->done[curr_prod_idx] = NULL; |
|
|
|
spin_unlock_bh(&cmdq->cmdq_lock); |
|
|
|
hinic_dump_ceq_info(cmdq->hwdev); |
|
return -ETIMEDOUT; |
|
} |
|
|
|
smp_rmb(); /* read error code after completion */ |
|
|
|
if (resp) { |
|
struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &curr_cmdq_wqe->wqe_lcmd; |
|
|
|
*resp = cpu_to_be64(wqe_lcmd->completion.direct_resp); |
|
} |
|
|
|
if (errcode != 0) |
|
return -EFAULT; |
|
|
|
return 0; |
|
} |
|
|
|
static int cmdq_set_arm_bit(struct hinic_cmdq *cmdq, void *buf_in, |
|
u16 in_size) |
|
{ |
|
struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe; |
|
u16 curr_prod_idx, next_prod_idx; |
|
struct hinic_wq *wq = cmdq->wq; |
|
struct hinic_hw_wqe *hw_wqe; |
|
int wrapped, num_wqebbs; |
|
|
|
/* Keep doorbell index correct */ |
|
spin_lock(&cmdq->cmdq_lock); |
|
|
|
/* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/ |
|
hw_wqe = hinic_get_wqe(wq, WQE_SCMD_SIZE, &curr_prod_idx); |
|
if (IS_ERR(hw_wqe)) { |
|
spin_unlock(&cmdq->cmdq_lock); |
|
return -EBUSY; |
|
} |
|
|
|
curr_cmdq_wqe = &hw_wqe->cmdq_wqe; |
|
|
|
wrapped = cmdq->wrapped; |
|
|
|
num_wqebbs = ALIGN(WQE_SCMD_SIZE, wq->wqebb_size) / wq->wqebb_size; |
|
next_prod_idx = curr_prod_idx + num_wqebbs; |
|
if (next_prod_idx >= wq->q_depth) { |
|
cmdq->wrapped = !cmdq->wrapped; |
|
next_prod_idx -= wq->q_depth; |
|
} |
|
|
|
cmdq_set_direct_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in, |
|
in_size, NULL, wrapped, HINIC_CMD_ACK_TYPE_CMDQ, |
|
HINIC_MOD_COMM, CMDQ_SET_ARM_CMD, curr_prod_idx); |
|
|
|
/* The data that is written to HW should be in Big Endian Format */ |
|
hinic_cpu_to_be32(&cmdq_wqe, WQE_SCMD_SIZE); |
|
|
|
/* cmdq wqe is not shadow, therefore wqe will be written to wq */ |
|
cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe); |
|
|
|
cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx); |
|
|
|
spin_unlock(&cmdq->cmdq_lock); |
|
return 0; |
|
} |
|
|
|
static int cmdq_params_valid(struct hinic_cmdq_buf *buf_in) |
|
{ |
|
if (buf_in->size > HINIC_CMDQ_MAX_DATA_SIZE) |
|
return -EINVAL; |
|
|
|
return 0; |
|
} |
|
|
|
/** |
|
* hinic_cmdq_direct_resp - send command with direct data as resp |
|
* @cmdqs: the cmdqs |
|
* @mod: module on the card that will handle the command |
|
* @cmd: the command |
|
* @buf_in: the buffer for the command |
|
* @resp: the response to return |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
int hinic_cmdq_direct_resp(struct hinic_cmdqs *cmdqs, |
|
enum hinic_mod_type mod, u8 cmd, |
|
struct hinic_cmdq_buf *buf_in, u64 *resp) |
|
{ |
|
struct hinic_hwif *hwif = cmdqs->hwif; |
|
struct pci_dev *pdev = hwif->pdev; |
|
int err; |
|
|
|
err = cmdq_params_valid(buf_in); |
|
if (err) { |
|
dev_err(&pdev->dev, "Invalid CMDQ parameters\n"); |
|
return err; |
|
} |
|
|
|
return cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC_CMDQ_SYNC], |
|
mod, cmd, buf_in, resp); |
|
} |
|
|
|
/** |
|
* hinic_set_arm_bit - set arm bit for enable interrupt again |
|
* @cmdqs: the cmdqs |
|
* @q_type: type of queue to set the arm bit for |
|
* @q_id: the queue number |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs, |
|
enum hinic_set_arm_qtype q_type, u32 q_id) |
|
{ |
|
struct hinic_cmdq *cmdq = &cmdqs->cmdq[HINIC_CMDQ_SYNC]; |
|
struct hinic_hwif *hwif = cmdqs->hwif; |
|
struct pci_dev *pdev = hwif->pdev; |
|
struct hinic_cmdq_arm_bit arm_bit; |
|
int err; |
|
|
|
arm_bit.q_type = q_type; |
|
arm_bit.q_id = q_id; |
|
|
|
err = cmdq_set_arm_bit(cmdq, &arm_bit, sizeof(arm_bit)); |
|
if (err) { |
|
dev_err(&pdev->dev, "Failed to set arm for qid %d\n", q_id); |
|
return err; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
static void clear_wqe_complete_bit(struct hinic_cmdq *cmdq, |
|
struct hinic_cmdq_wqe *wqe) |
|
{ |
|
u32 header_info = be32_to_cpu(CMDQ_WQE_HEADER(wqe)->header_info); |
|
unsigned int bufdesc_len, wqe_size; |
|
struct hinic_ctrl *ctrl; |
|
|
|
bufdesc_len = HINIC_CMDQ_WQE_HEADER_GET(header_info, BUFDESC_LEN); |
|
wqe_size = cmdq_wqe_size_from_bdlen(bufdesc_len); |
|
if (wqe_size == WQE_LCMD_SIZE) { |
|
struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd; |
|
|
|
ctrl = &wqe_lcmd->ctrl; |
|
} else { |
|
struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; |
|
struct hinic_cmdq_wqe_scmd *wqe_scmd; |
|
|
|
wqe_scmd = &direct_wqe->wqe_scmd; |
|
ctrl = &wqe_scmd->ctrl; |
|
} |
|
|
|
/* clear HW busy bit */ |
|
ctrl->ctrl_info = 0; |
|
|
|
wmb(); /* verify wqe is clear */ |
|
} |
|
|
|
/** |
|
* cmdq_arm_ceq_handler - cmdq completion event handler for arm command |
|
* @cmdq: the cmdq of the arm command |
|
* @wqe: the wqe of the arm command |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
static int cmdq_arm_ceq_handler(struct hinic_cmdq *cmdq, |
|
struct hinic_cmdq_wqe *wqe) |
|
{ |
|
struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe; |
|
struct hinic_cmdq_wqe_scmd *wqe_scmd; |
|
struct hinic_ctrl *ctrl; |
|
u32 ctrl_info; |
|
|
|
wqe_scmd = &direct_wqe->wqe_scmd; |
|
ctrl = &wqe_scmd->ctrl; |
|
ctrl_info = be32_to_cpu(ctrl->ctrl_info); |
|
|
|
/* HW should toggle the HW BUSY BIT */ |
|
if (!CMDQ_WQE_COMPLETED(ctrl_info)) |
|
return -EBUSY; |
|
|
|
clear_wqe_complete_bit(cmdq, wqe); |
|
|
|
hinic_put_wqe(cmdq->wq, WQE_SCMD_SIZE); |
|
return 0; |
|
} |
|
|
|
static void cmdq_update_errcode(struct hinic_cmdq *cmdq, u16 prod_idx, |
|
int errcode) |
|
{ |
|
if (cmdq->errcode[prod_idx]) |
|
*cmdq->errcode[prod_idx] = errcode; |
|
} |
|
|
|
/** |
|
* cmdq_arm_ceq_handler - cmdq completion event handler for sync command |
|
* @cmdq: the cmdq of the command |
|
* @cons_idx: the consumer index to update the error code for |
|
* @errcode: the error code |
|
**/ |
|
static void cmdq_sync_cmd_handler(struct hinic_cmdq *cmdq, u16 cons_idx, |
|
int errcode) |
|
{ |
|
u16 prod_idx = cons_idx; |
|
|
|
spin_lock(&cmdq->cmdq_lock); |
|
cmdq_update_errcode(cmdq, prod_idx, errcode); |
|
|
|
wmb(); /* write all before update for the command request */ |
|
|
|
if (cmdq->done[prod_idx]) |
|
complete(cmdq->done[prod_idx]); |
|
spin_unlock(&cmdq->cmdq_lock); |
|
} |
|
|
|
static int cmdq_cmd_ceq_handler(struct hinic_cmdq *cmdq, u16 ci, |
|
struct hinic_cmdq_wqe *cmdq_wqe) |
|
{ |
|
struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &cmdq_wqe->wqe_lcmd; |
|
struct hinic_status *status = &wqe_lcmd->status; |
|
struct hinic_ctrl *ctrl = &wqe_lcmd->ctrl; |
|
int errcode; |
|
|
|
if (!CMDQ_WQE_COMPLETED(be32_to_cpu(ctrl->ctrl_info))) |
|
return -EBUSY; |
|
|
|
dma_rmb(); |
|
|
|
errcode = CMDQ_WQE_ERRCODE_GET(be32_to_cpu(status->status_info), VAL); |
|
|
|
cmdq_sync_cmd_handler(cmdq, ci, errcode); |
|
|
|
clear_wqe_complete_bit(cmdq, cmdq_wqe); |
|
hinic_put_wqe(cmdq->wq, WQE_LCMD_SIZE); |
|
return 0; |
|
} |
|
|
|
/** |
|
* cmdq_ceq_handler - cmdq completion event handler |
|
* @handle: private data for the handler(cmdqs) |
|
* @ceqe_data: ceq element data |
|
**/ |
|
static void cmdq_ceq_handler(void *handle, u32 ceqe_data) |
|
{ |
|
enum hinic_cmdq_type cmdq_type = CMDQ_CEQE_GET(ceqe_data, TYPE); |
|
struct hinic_cmdqs *cmdqs = (struct hinic_cmdqs *)handle; |
|
struct hinic_cmdq *cmdq = &cmdqs->cmdq[cmdq_type]; |
|
struct hinic_cmdq_header *header; |
|
struct hinic_hw_wqe *hw_wqe; |
|
int err, set_arm = 0; |
|
u32 saved_data; |
|
u16 ci; |
|
|
|
/* Read the smallest wqe size for getting wqe size */ |
|
while ((hw_wqe = hinic_read_wqe(cmdq->wq, WQE_SCMD_SIZE, &ci))) { |
|
if (IS_ERR(hw_wqe)) |
|
break; |
|
|
|
header = CMDQ_WQE_HEADER(&hw_wqe->cmdq_wqe); |
|
saved_data = be32_to_cpu(header->saved_data); |
|
|
|
if (HINIC_SAVED_DATA_GET(saved_data, ARM)) { |
|
/* arm_bit was set until here */ |
|
set_arm = 0; |
|
|
|
if (cmdq_arm_ceq_handler(cmdq, &hw_wqe->cmdq_wqe)) |
|
break; |
|
} else { |
|
set_arm = 1; |
|
|
|
hw_wqe = hinic_read_wqe(cmdq->wq, WQE_LCMD_SIZE, &ci); |
|
if (IS_ERR(hw_wqe)) |
|
break; |
|
|
|
if (cmdq_cmd_ceq_handler(cmdq, ci, &hw_wqe->cmdq_wqe)) |
|
break; |
|
} |
|
} |
|
|
|
if (set_arm) { |
|
struct hinic_hwif *hwif = cmdqs->hwif; |
|
struct pci_dev *pdev = hwif->pdev; |
|
|
|
err = hinic_set_arm_bit(cmdqs, HINIC_SET_ARM_CMDQ, cmdq_type); |
|
if (err) |
|
dev_err(&pdev->dev, "Failed to set arm for CMDQ\n"); |
|
} |
|
} |
|
|
|
/** |
|
* cmdq_init_queue_ctxt - init the queue ctxt of a cmdq |
|
* @cmdq_ctxt: cmdq ctxt to initialize |
|
* @cmdq: the cmdq |
|
* @cmdq_pages: the memory of the queue |
|
**/ |
|
static void cmdq_init_queue_ctxt(struct hinic_cmdq_ctxt *cmdq_ctxt, |
|
struct hinic_cmdq *cmdq, |
|
struct hinic_cmdq_pages *cmdq_pages) |
|
{ |
|
struct hinic_cmdq_ctxt_info *ctxt_info = &cmdq_ctxt->ctxt_info; |
|
u64 wq_first_page_paddr, cmdq_first_block_paddr, pfn; |
|
struct hinic_cmdqs *cmdqs = cmdq_to_cmdqs(cmdq); |
|
struct hinic_wq *wq = cmdq->wq; |
|
|
|
/* The data in the HW is in Big Endian Format */ |
|
wq_first_page_paddr = be64_to_cpu(*wq->block_vaddr); |
|
|
|
pfn = CMDQ_PFN(wq_first_page_paddr, SZ_4K); |
|
|
|
ctxt_info->curr_wqe_page_pfn = |
|
HINIC_CMDQ_CTXT_PAGE_INFO_SET(pfn, CURR_WQE_PAGE_PFN) | |
|
HINIC_CMDQ_CTXT_PAGE_INFO_SET(HINIC_CEQ_ID_CMDQ, EQ_ID) | |
|
HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_ARM) | |
|
HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_EN) | |
|
HINIC_CMDQ_CTXT_PAGE_INFO_SET(cmdq->wrapped, WRAPPED); |
|
|
|
if (wq->num_q_pages != 1) { |
|
/* block PFN - Read Modify Write */ |
|
cmdq_first_block_paddr = cmdq_pages->page_paddr; |
|
|
|
pfn = CMDQ_PFN(cmdq_first_block_paddr, wq->wq_page_size); |
|
} |
|
|
|
ctxt_info->wq_block_pfn = |
|
HINIC_CMDQ_CTXT_BLOCK_INFO_SET(pfn, WQ_BLOCK_PFN) | |
|
HINIC_CMDQ_CTXT_BLOCK_INFO_SET(atomic_read(&wq->cons_idx), CI); |
|
|
|
cmdq_ctxt->func_idx = HINIC_HWIF_FUNC_IDX(cmdqs->hwif); |
|
cmdq_ctxt->ppf_idx = HINIC_HWIF_PPF_IDX(cmdqs->hwif); |
|
cmdq_ctxt->cmdq_type = cmdq->cmdq_type; |
|
} |
|
|
|
/** |
|
* init_cmdq - initialize cmdq |
|
* @cmdq: the cmdq |
|
* @wq: the wq attaced to the cmdq |
|
* @q_type: the cmdq type of the cmdq |
|
* @db_area: doorbell area for the cmdq |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
static int init_cmdq(struct hinic_cmdq *cmdq, struct hinic_wq *wq, |
|
enum hinic_cmdq_type q_type, void __iomem *db_area) |
|
{ |
|
int err; |
|
|
|
cmdq->wq = wq; |
|
cmdq->cmdq_type = q_type; |
|
cmdq->wrapped = 1; |
|
|
|
spin_lock_init(&cmdq->cmdq_lock); |
|
|
|
cmdq->done = vzalloc(array_size(sizeof(*cmdq->done), wq->q_depth)); |
|
if (!cmdq->done) |
|
return -ENOMEM; |
|
|
|
cmdq->errcode = vzalloc(array_size(sizeof(*cmdq->errcode), |
|
wq->q_depth)); |
|
if (!cmdq->errcode) { |
|
err = -ENOMEM; |
|
goto err_errcode; |
|
} |
|
|
|
cmdq->db_base = db_area + CMDQ_DB_OFF; |
|
return 0; |
|
|
|
err_errcode: |
|
vfree(cmdq->done); |
|
return err; |
|
} |
|
|
|
/** |
|
* free_cmdq - Free cmdq |
|
* @cmdq: the cmdq to free |
|
**/ |
|
static void free_cmdq(struct hinic_cmdq *cmdq) |
|
{ |
|
vfree(cmdq->errcode); |
|
vfree(cmdq->done); |
|
} |
|
|
|
/** |
|
* init_cmdqs_ctxt - write the cmdq ctxt to HW after init all cmdq |
|
* @hwdev: the NIC HW device |
|
* @cmdqs: cmdqs to write the ctxts for |
|
* @db_area: db_area for all the cmdqs |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
static int init_cmdqs_ctxt(struct hinic_hwdev *hwdev, |
|
struct hinic_cmdqs *cmdqs, void __iomem **db_area) |
|
{ |
|
struct hinic_hwif *hwif = hwdev->hwif; |
|
enum hinic_cmdq_type type, cmdq_type; |
|
struct hinic_cmdq_ctxt *cmdq_ctxts; |
|
struct pci_dev *pdev = hwif->pdev; |
|
struct hinic_pfhwdev *pfhwdev; |
|
size_t cmdq_ctxts_size; |
|
int err; |
|
|
|
cmdq_ctxts_size = HINIC_MAX_CMDQ_TYPES * sizeof(*cmdq_ctxts); |
|
cmdq_ctxts = devm_kzalloc(&pdev->dev, cmdq_ctxts_size, GFP_KERNEL); |
|
if (!cmdq_ctxts) |
|
return -ENOMEM; |
|
|
|
pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); |
|
|
|
cmdq_type = HINIC_CMDQ_SYNC; |
|
for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) { |
|
cmdqs->cmdq[cmdq_type].hwdev = hwdev; |
|
err = init_cmdq(&cmdqs->cmdq[cmdq_type], |
|
&cmdqs->saved_wqs[cmdq_type], cmdq_type, |
|
db_area[cmdq_type]); |
|
if (err) { |
|
dev_err(&pdev->dev, "Failed to initialize cmdq\n"); |
|
goto err_init_cmdq; |
|
} |
|
|
|
cmdq_init_queue_ctxt(&cmdq_ctxts[cmdq_type], |
|
&cmdqs->cmdq[cmdq_type], |
|
&cmdqs->cmdq_pages); |
|
} |
|
|
|
/* Write the CMDQ ctxts */ |
|
cmdq_type = HINIC_CMDQ_SYNC; |
|
for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) { |
|
err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, |
|
HINIC_COMM_CMD_CMDQ_CTXT_SET, |
|
&cmdq_ctxts[cmdq_type], |
|
sizeof(cmdq_ctxts[cmdq_type]), |
|
NULL, NULL, HINIC_MGMT_MSG_SYNC); |
|
if (err) { |
|
dev_err(&pdev->dev, "Failed to set CMDQ CTXT type = %d\n", |
|
cmdq_type); |
|
goto err_write_cmdq_ctxt; |
|
} |
|
} |
|
|
|
devm_kfree(&pdev->dev, cmdq_ctxts); |
|
return 0; |
|
|
|
err_write_cmdq_ctxt: |
|
cmdq_type = HINIC_MAX_CMDQ_TYPES; |
|
|
|
err_init_cmdq: |
|
for (type = HINIC_CMDQ_SYNC; type < cmdq_type; type++) |
|
free_cmdq(&cmdqs->cmdq[type]); |
|
|
|
devm_kfree(&pdev->dev, cmdq_ctxts); |
|
return err; |
|
} |
|
|
|
static int hinic_set_cmdq_depth(struct hinic_hwdev *hwdev, u16 cmdq_depth) |
|
{ |
|
struct hinic_cmd_hw_ioctxt hw_ioctxt = { 0 }; |
|
struct hinic_pfhwdev *pfhwdev; |
|
|
|
pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev); |
|
|
|
hw_ioctxt.func_idx = HINIC_HWIF_FUNC_IDX(hwdev->hwif); |
|
hw_ioctxt.ppf_idx = HINIC_HWIF_PPF_IDX(hwdev->hwif); |
|
|
|
hw_ioctxt.set_cmdq_depth = HW_IOCTXT_SET_CMDQ_DEPTH_ENABLE; |
|
hw_ioctxt.cmdq_depth = (u8)ilog2(cmdq_depth); |
|
|
|
return hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, |
|
HINIC_COMM_CMD_HWCTXT_SET, |
|
&hw_ioctxt, sizeof(hw_ioctxt), NULL, |
|
NULL, HINIC_MGMT_MSG_SYNC); |
|
} |
|
|
|
/** |
|
* hinic_init_cmdqs - init all cmdqs |
|
* @cmdqs: cmdqs to init |
|
* @hwif: HW interface for accessing cmdqs |
|
* @db_area: doorbell areas for all the cmdqs |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif, |
|
void __iomem **db_area) |
|
{ |
|
struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs); |
|
struct pci_dev *pdev = hwif->pdev; |
|
struct hinic_hwdev *hwdev; |
|
size_t saved_wqs_size; |
|
u16 max_wqe_size; |
|
int err; |
|
|
|
cmdqs->hwif = hwif; |
|
cmdqs->cmdq_buf_pool = dma_pool_create("hinic_cmdq", &pdev->dev, |
|
HINIC_CMDQ_BUF_SIZE, |
|
HINIC_CMDQ_BUF_SIZE, 0); |
|
if (!cmdqs->cmdq_buf_pool) |
|
return -ENOMEM; |
|
|
|
saved_wqs_size = HINIC_MAX_CMDQ_TYPES * sizeof(struct hinic_wq); |
|
cmdqs->saved_wqs = devm_kzalloc(&pdev->dev, saved_wqs_size, GFP_KERNEL); |
|
if (!cmdqs->saved_wqs) { |
|
err = -ENOMEM; |
|
goto err_saved_wqs; |
|
} |
|
|
|
max_wqe_size = WQE_LCMD_SIZE; |
|
err = hinic_wqs_cmdq_alloc(&cmdqs->cmdq_pages, cmdqs->saved_wqs, hwif, |
|
HINIC_MAX_CMDQ_TYPES, CMDQ_WQEBB_SIZE, |
|
CMDQ_WQ_PAGE_SIZE, CMDQ_DEPTH, max_wqe_size); |
|
if (err) { |
|
dev_err(&pdev->dev, "Failed to allocate CMDQ wqs\n"); |
|
goto err_cmdq_wqs; |
|
} |
|
|
|
hwdev = container_of(func_to_io, struct hinic_hwdev, func_to_io); |
|
err = init_cmdqs_ctxt(hwdev, cmdqs, db_area); |
|
if (err) { |
|
dev_err(&pdev->dev, "Failed to write cmdq ctxt\n"); |
|
goto err_cmdq_ctxt; |
|
} |
|
|
|
hinic_ceq_register_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ, cmdqs, |
|
cmdq_ceq_handler); |
|
|
|
err = hinic_set_cmdq_depth(hwdev, CMDQ_DEPTH); |
|
if (err) { |
|
dev_err(&hwif->pdev->dev, "Failed to set cmdq depth\n"); |
|
goto err_set_cmdq_depth; |
|
} |
|
|
|
return 0; |
|
|
|
err_set_cmdq_depth: |
|
hinic_ceq_unregister_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ); |
|
|
|
err_cmdq_ctxt: |
|
hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs, |
|
HINIC_MAX_CMDQ_TYPES); |
|
|
|
err_cmdq_wqs: |
|
devm_kfree(&pdev->dev, cmdqs->saved_wqs); |
|
|
|
err_saved_wqs: |
|
dma_pool_destroy(cmdqs->cmdq_buf_pool); |
|
return err; |
|
} |
|
|
|
/** |
|
* hinic_free_cmdqs - free all cmdqs |
|
* @cmdqs: cmdqs to free |
|
**/ |
|
void hinic_free_cmdqs(struct hinic_cmdqs *cmdqs) |
|
{ |
|
struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs); |
|
struct hinic_hwif *hwif = cmdqs->hwif; |
|
struct pci_dev *pdev = hwif->pdev; |
|
enum hinic_cmdq_type cmdq_type; |
|
|
|
hinic_ceq_unregister_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ); |
|
|
|
cmdq_type = HINIC_CMDQ_SYNC; |
|
for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) |
|
free_cmdq(&cmdqs->cmdq[cmdq_type]); |
|
|
|
hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs, |
|
HINIC_MAX_CMDQ_TYPES); |
|
|
|
devm_kfree(&pdev->dev, cmdqs->saved_wqs); |
|
|
|
dma_pool_destroy(cmdqs->cmdq_buf_pool); |
|
}
|
|
|