forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
904 lines
23 KiB
904 lines
23 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* |
|
* Huawei HiNIC PCI Express Linux driver |
|
* Copyright(c) 2017 Huawei Technologies Co., Ltd |
|
*/ |
|
|
|
#include <linux/kernel.h> |
|
#include <linux/types.h> |
|
#include <linux/pci.h> |
|
#include <linux/device.h> |
|
#include <linux/dma-mapping.h> |
|
#include <linux/slab.h> |
|
#include <linux/atomic.h> |
|
#include <linux/semaphore.h> |
|
#include <linux/errno.h> |
|
#include <linux/vmalloc.h> |
|
#include <linux/err.h> |
|
#include <asm/byteorder.h> |
|
|
|
#include "hinic_hw_if.h" |
|
#include "hinic_hw_wqe.h" |
|
#include "hinic_hw_wq.h" |
|
#include "hinic_hw_cmdq.h" |
|
|
|
#define WQS_BLOCKS_PER_PAGE 4 |
|
|
|
#define WQ_BLOCK_SIZE 4096 |
|
#define WQS_PAGE_SIZE (WQS_BLOCKS_PER_PAGE * WQ_BLOCK_SIZE) |
|
|
|
#define WQS_MAX_NUM_BLOCKS 128 |
|
#define WQS_FREE_BLOCKS_SIZE(wqs) (WQS_MAX_NUM_BLOCKS * \ |
|
sizeof((wqs)->free_blocks[0])) |
|
|
|
#define WQ_SIZE(wq) ((wq)->q_depth * (wq)->wqebb_size) |
|
|
|
#define WQ_PAGE_ADDR_SIZE sizeof(u64) |
|
#define WQ_MAX_PAGES (WQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE) |
|
|
|
#define CMDQ_BLOCK_SIZE 512 |
|
#define CMDQ_PAGE_SIZE 4096 |
|
|
|
#define CMDQ_WQ_MAX_PAGES (CMDQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE) |
|
|
|
#define WQ_BASE_VADDR(wqs, wq) \ |
|
((void *)((wqs)->page_vaddr[(wq)->page_idx]) \ |
|
+ (wq)->block_idx * WQ_BLOCK_SIZE) |
|
|
|
#define WQ_BASE_PADDR(wqs, wq) \ |
|
((wqs)->page_paddr[(wq)->page_idx] \ |
|
+ (wq)->block_idx * WQ_BLOCK_SIZE) |
|
|
|
#define WQ_BASE_ADDR(wqs, wq) \ |
|
((void *)((wqs)->shadow_page_vaddr[(wq)->page_idx]) \ |
|
+ (wq)->block_idx * WQ_BLOCK_SIZE) |
|
|
|
#define CMDQ_BASE_VADDR(cmdq_pages, wq) \ |
|
((void *)((cmdq_pages)->page_vaddr) \ |
|
+ (wq)->block_idx * CMDQ_BLOCK_SIZE) |
|
|
|
#define CMDQ_BASE_PADDR(cmdq_pages, wq) \ |
|
((cmdq_pages)->page_paddr \ |
|
+ (wq)->block_idx * CMDQ_BLOCK_SIZE) |
|
|
|
#define CMDQ_BASE_ADDR(cmdq_pages, wq) \ |
|
((void *)((cmdq_pages)->shadow_page_vaddr) \ |
|
+ (wq)->block_idx * CMDQ_BLOCK_SIZE) |
|
|
|
#define WQ_PAGE_ADDR(wq, idx) \ |
|
((wq)->shadow_block_vaddr[WQE_PAGE_NUM(wq, idx)]) |
|
|
|
#define MASKED_WQE_IDX(wq, idx) ((idx) & (wq)->mask) |
|
|
|
#define WQE_IN_RANGE(wqe, start, end) \ |
|
(((unsigned long)(wqe) >= (unsigned long)(start)) && \ |
|
((unsigned long)(wqe) < (unsigned long)(end))) |
|
|
|
#define WQE_SHADOW_PAGE(wq, wqe) \ |
|
(((unsigned long)(wqe) - (unsigned long)(wq)->shadow_wqe) \ |
|
/ (wq)->max_wqe_size) |
|
|
|
static inline int WQE_PAGE_OFF(struct hinic_wq *wq, u16 idx) |
|
{ |
|
return (((idx) & ((wq)->num_wqebbs_per_page - 1)) |
|
<< (wq)->wqebb_size_shift); |
|
} |
|
|
|
static inline int WQE_PAGE_NUM(struct hinic_wq *wq, u16 idx) |
|
{ |
|
return (((idx) >> ((wq)->wqebbs_per_page_shift)) |
|
& ((wq)->num_q_pages - 1)); |
|
} |
|
|
|
/** |
|
* queue_alloc_page - allocate page for Queue |
|
* @hwif: HW interface for allocating DMA |
|
* @vaddr: virtual address will be returned in this address |
|
* @paddr: physical address will be returned in this address |
|
* @shadow_vaddr: VM area will be return here for holding WQ page addresses |
|
* @page_sz: page size of each WQ page |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
static int queue_alloc_page(struct hinic_hwif *hwif, u64 **vaddr, u64 *paddr, |
|
void ***shadow_vaddr, size_t page_sz) |
|
{ |
|
struct pci_dev *pdev = hwif->pdev; |
|
dma_addr_t dma_addr; |
|
|
|
*vaddr = dma_alloc_coherent(&pdev->dev, page_sz, &dma_addr, |
|
GFP_KERNEL); |
|
if (!*vaddr) { |
|
dev_err(&pdev->dev, "Failed to allocate dma for wqs page\n"); |
|
return -ENOMEM; |
|
} |
|
|
|
*paddr = (u64)dma_addr; |
|
|
|
/* use vzalloc for big mem */ |
|
*shadow_vaddr = vzalloc(page_sz); |
|
if (!*shadow_vaddr) |
|
goto err_shadow_vaddr; |
|
|
|
return 0; |
|
|
|
err_shadow_vaddr: |
|
dma_free_coherent(&pdev->dev, page_sz, *vaddr, dma_addr); |
|
return -ENOMEM; |
|
} |
|
|
|
/** |
|
* wqs_allocate_page - allocate page for WQ set |
|
* @wqs: Work Queue Set |
|
* @page_idx: the page index of the page will be allocated |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
static int wqs_allocate_page(struct hinic_wqs *wqs, int page_idx) |
|
{ |
|
return queue_alloc_page(wqs->hwif, &wqs->page_vaddr[page_idx], |
|
&wqs->page_paddr[page_idx], |
|
&wqs->shadow_page_vaddr[page_idx], |
|
WQS_PAGE_SIZE); |
|
} |
|
|
|
/** |
|
* wqs_free_page - free page of WQ set |
|
* @wqs: Work Queue Set |
|
* @page_idx: the page index of the page will be freed |
|
**/ |
|
static void wqs_free_page(struct hinic_wqs *wqs, int page_idx) |
|
{ |
|
struct hinic_hwif *hwif = wqs->hwif; |
|
struct pci_dev *pdev = hwif->pdev; |
|
|
|
dma_free_coherent(&pdev->dev, WQS_PAGE_SIZE, |
|
wqs->page_vaddr[page_idx], |
|
(dma_addr_t)wqs->page_paddr[page_idx]); |
|
vfree(wqs->shadow_page_vaddr[page_idx]); |
|
} |
|
|
|
/** |
|
* cmdq_allocate_page - allocate page for cmdq |
|
* @cmdq_pages: the pages of the cmdq queue struct to hold the page |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
static int cmdq_allocate_page(struct hinic_cmdq_pages *cmdq_pages) |
|
{ |
|
return queue_alloc_page(cmdq_pages->hwif, &cmdq_pages->page_vaddr, |
|
&cmdq_pages->page_paddr, |
|
&cmdq_pages->shadow_page_vaddr, |
|
CMDQ_PAGE_SIZE); |
|
} |
|
|
|
/** |
|
* cmdq_free_page - free page from cmdq |
|
* @cmdq_pages: the pages of the cmdq queue struct that hold the page |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
static void cmdq_free_page(struct hinic_cmdq_pages *cmdq_pages) |
|
{ |
|
struct hinic_hwif *hwif = cmdq_pages->hwif; |
|
struct pci_dev *pdev = hwif->pdev; |
|
|
|
dma_free_coherent(&pdev->dev, CMDQ_PAGE_SIZE, |
|
cmdq_pages->page_vaddr, |
|
(dma_addr_t)cmdq_pages->page_paddr); |
|
vfree(cmdq_pages->shadow_page_vaddr); |
|
} |
|
|
|
static int alloc_page_arrays(struct hinic_wqs *wqs) |
|
{ |
|
struct hinic_hwif *hwif = wqs->hwif; |
|
struct pci_dev *pdev = hwif->pdev; |
|
size_t size; |
|
|
|
size = wqs->num_pages * sizeof(*wqs->page_paddr); |
|
wqs->page_paddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); |
|
if (!wqs->page_paddr) |
|
return -ENOMEM; |
|
|
|
size = wqs->num_pages * sizeof(*wqs->page_vaddr); |
|
wqs->page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); |
|
if (!wqs->page_vaddr) |
|
goto err_page_vaddr; |
|
|
|
size = wqs->num_pages * sizeof(*wqs->shadow_page_vaddr); |
|
wqs->shadow_page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); |
|
if (!wqs->shadow_page_vaddr) |
|
goto err_page_shadow_vaddr; |
|
|
|
return 0; |
|
|
|
err_page_shadow_vaddr: |
|
devm_kfree(&pdev->dev, wqs->page_vaddr); |
|
|
|
err_page_vaddr: |
|
devm_kfree(&pdev->dev, wqs->page_paddr); |
|
return -ENOMEM; |
|
} |
|
|
|
static void free_page_arrays(struct hinic_wqs *wqs) |
|
{ |
|
struct hinic_hwif *hwif = wqs->hwif; |
|
struct pci_dev *pdev = hwif->pdev; |
|
|
|
devm_kfree(&pdev->dev, wqs->shadow_page_vaddr); |
|
devm_kfree(&pdev->dev, wqs->page_vaddr); |
|
devm_kfree(&pdev->dev, wqs->page_paddr); |
|
} |
|
|
|
static int wqs_next_block(struct hinic_wqs *wqs, int *page_idx, |
|
int *block_idx) |
|
{ |
|
int pos; |
|
|
|
down(&wqs->alloc_blocks_lock); |
|
|
|
wqs->num_free_blks--; |
|
|
|
if (wqs->num_free_blks < 0) { |
|
wqs->num_free_blks++; |
|
up(&wqs->alloc_blocks_lock); |
|
return -ENOMEM; |
|
} |
|
|
|
pos = wqs->alloc_blk_pos++; |
|
pos &= WQS_MAX_NUM_BLOCKS - 1; |
|
|
|
*page_idx = wqs->free_blocks[pos].page_idx; |
|
*block_idx = wqs->free_blocks[pos].block_idx; |
|
|
|
wqs->free_blocks[pos].page_idx = -1; |
|
wqs->free_blocks[pos].block_idx = -1; |
|
|
|
up(&wqs->alloc_blocks_lock); |
|
return 0; |
|
} |
|
|
|
static void wqs_return_block(struct hinic_wqs *wqs, int page_idx, |
|
int block_idx) |
|
{ |
|
int pos; |
|
|
|
down(&wqs->alloc_blocks_lock); |
|
|
|
pos = wqs->return_blk_pos++; |
|
pos &= WQS_MAX_NUM_BLOCKS - 1; |
|
|
|
wqs->free_blocks[pos].page_idx = page_idx; |
|
wqs->free_blocks[pos].block_idx = block_idx; |
|
|
|
wqs->num_free_blks++; |
|
|
|
up(&wqs->alloc_blocks_lock); |
|
} |
|
|
|
static void init_wqs_blocks_arr(struct hinic_wqs *wqs) |
|
{ |
|
int page_idx, blk_idx, pos = 0; |
|
|
|
for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) { |
|
for (blk_idx = 0; blk_idx < WQS_BLOCKS_PER_PAGE; blk_idx++) { |
|
wqs->free_blocks[pos].page_idx = page_idx; |
|
wqs->free_blocks[pos].block_idx = blk_idx; |
|
pos++; |
|
} |
|
} |
|
|
|
wqs->alloc_blk_pos = 0; |
|
wqs->return_blk_pos = pos; |
|
wqs->num_free_blks = pos; |
|
|
|
sema_init(&wqs->alloc_blocks_lock, 1); |
|
} |
|
|
|
/** |
|
* hinic_wqs_alloc - allocate Work Queues set |
|
* @wqs: Work Queue Set |
|
* @max_wqs: maximum wqs to allocate |
|
* @hwif: HW interface for use for the allocation |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
int hinic_wqs_alloc(struct hinic_wqs *wqs, int max_wqs, |
|
struct hinic_hwif *hwif) |
|
{ |
|
struct pci_dev *pdev = hwif->pdev; |
|
int err, i, page_idx; |
|
|
|
max_wqs = ALIGN(max_wqs, WQS_BLOCKS_PER_PAGE); |
|
if (max_wqs > WQS_MAX_NUM_BLOCKS) { |
|
dev_err(&pdev->dev, "Invalid max_wqs = %d\n", max_wqs); |
|
return -EINVAL; |
|
} |
|
|
|
wqs->hwif = hwif; |
|
wqs->num_pages = max_wqs / WQS_BLOCKS_PER_PAGE; |
|
|
|
if (alloc_page_arrays(wqs)) { |
|
dev_err(&pdev->dev, |
|
"Failed to allocate mem for page addresses\n"); |
|
return -ENOMEM; |
|
} |
|
|
|
for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) { |
|
err = wqs_allocate_page(wqs, page_idx); |
|
if (err) { |
|
dev_err(&pdev->dev, "Failed wq page allocation\n"); |
|
goto err_wq_allocate_page; |
|
} |
|
} |
|
|
|
wqs->free_blocks = devm_kzalloc(&pdev->dev, WQS_FREE_BLOCKS_SIZE(wqs), |
|
GFP_KERNEL); |
|
if (!wqs->free_blocks) { |
|
err = -ENOMEM; |
|
goto err_alloc_blocks; |
|
} |
|
|
|
init_wqs_blocks_arr(wqs); |
|
return 0; |
|
|
|
err_alloc_blocks: |
|
err_wq_allocate_page: |
|
for (i = 0; i < page_idx; i++) |
|
wqs_free_page(wqs, i); |
|
|
|
free_page_arrays(wqs); |
|
return err; |
|
} |
|
|
|
/** |
|
* hinic_wqs_free - free Work Queues set |
|
* @wqs: Work Queue Set |
|
**/ |
|
void hinic_wqs_free(struct hinic_wqs *wqs) |
|
{ |
|
struct hinic_hwif *hwif = wqs->hwif; |
|
struct pci_dev *pdev = hwif->pdev; |
|
int page_idx; |
|
|
|
devm_kfree(&pdev->dev, wqs->free_blocks); |
|
|
|
for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) |
|
wqs_free_page(wqs, page_idx); |
|
|
|
free_page_arrays(wqs); |
|
} |
|
|
|
/** |
|
* alloc_wqes_shadow - allocate WQE shadows for WQ |
|
* @wq: WQ to allocate shadows for |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
static int alloc_wqes_shadow(struct hinic_wq *wq) |
|
{ |
|
struct hinic_hwif *hwif = wq->hwif; |
|
struct pci_dev *pdev = hwif->pdev; |
|
size_t size; |
|
|
|
size = wq->num_q_pages * wq->max_wqe_size; |
|
wq->shadow_wqe = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); |
|
if (!wq->shadow_wqe) |
|
return -ENOMEM; |
|
|
|
size = wq->num_q_pages * sizeof(wq->prod_idx); |
|
wq->shadow_idx = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); |
|
if (!wq->shadow_idx) |
|
goto err_shadow_idx; |
|
|
|
return 0; |
|
|
|
err_shadow_idx: |
|
devm_kfree(&pdev->dev, wq->shadow_wqe); |
|
return -ENOMEM; |
|
} |
|
|
|
/** |
|
* free_wqes_shadow - free WQE shadows of WQ |
|
* @wq: WQ to free shadows from |
|
**/ |
|
static void free_wqes_shadow(struct hinic_wq *wq) |
|
{ |
|
struct hinic_hwif *hwif = wq->hwif; |
|
struct pci_dev *pdev = hwif->pdev; |
|
|
|
devm_kfree(&pdev->dev, wq->shadow_idx); |
|
devm_kfree(&pdev->dev, wq->shadow_wqe); |
|
} |
|
|
|
/** |
|
* free_wq_pages - free pages of WQ |
|
* @hwif: HW interface for releasing dma addresses |
|
* @wq: WQ to free pages from |
|
* @num_q_pages: number pages to free |
|
**/ |
|
static void free_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif, |
|
int num_q_pages) |
|
{ |
|
struct pci_dev *pdev = hwif->pdev; |
|
int i; |
|
|
|
for (i = 0; i < num_q_pages; i++) { |
|
void **vaddr = &wq->shadow_block_vaddr[i]; |
|
u64 *paddr = &wq->block_vaddr[i]; |
|
dma_addr_t dma_addr; |
|
|
|
dma_addr = (dma_addr_t)be64_to_cpu(*paddr); |
|
dma_free_coherent(&pdev->dev, wq->wq_page_size, *vaddr, |
|
dma_addr); |
|
} |
|
|
|
free_wqes_shadow(wq); |
|
} |
|
|
|
/** |
|
* alloc_wq_pages - alloc pages for WQ |
|
* @hwif: HW interface for allocating dma addresses |
|
* @wq: WQ to allocate pages for |
|
* @max_pages: maximum pages allowed |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
static int alloc_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif, |
|
int max_pages) |
|
{ |
|
struct pci_dev *pdev = hwif->pdev; |
|
int i, err, num_q_pages; |
|
|
|
num_q_pages = ALIGN(WQ_SIZE(wq), wq->wq_page_size) / wq->wq_page_size; |
|
if (num_q_pages > max_pages) { |
|
dev_err(&pdev->dev, "Number wq pages exceeds the limit\n"); |
|
return -EINVAL; |
|
} |
|
|
|
if (num_q_pages & (num_q_pages - 1)) { |
|
dev_err(&pdev->dev, "Number wq pages must be power of 2\n"); |
|
return -EINVAL; |
|
} |
|
|
|
wq->num_q_pages = num_q_pages; |
|
|
|
err = alloc_wqes_shadow(wq); |
|
if (err) { |
|
dev_err(&pdev->dev, "Failed to allocate wqe shadow\n"); |
|
return err; |
|
} |
|
|
|
for (i = 0; i < num_q_pages; i++) { |
|
void **vaddr = &wq->shadow_block_vaddr[i]; |
|
u64 *paddr = &wq->block_vaddr[i]; |
|
dma_addr_t dma_addr; |
|
|
|
*vaddr = dma_alloc_coherent(&pdev->dev, wq->wq_page_size, |
|
&dma_addr, GFP_KERNEL); |
|
if (!*vaddr) { |
|
dev_err(&pdev->dev, "Failed to allocate wq page\n"); |
|
goto err_alloc_wq_pages; |
|
} |
|
|
|
/* HW uses Big Endian Format */ |
|
*paddr = cpu_to_be64(dma_addr); |
|
} |
|
|
|
return 0; |
|
|
|
err_alloc_wq_pages: |
|
free_wq_pages(wq, hwif, i); |
|
return -ENOMEM; |
|
} |
|
|
|
/** |
|
* hinic_wq_allocate - Allocate the WQ resources from the WQS |
|
* @wqs: WQ set from which to allocate the WQ resources |
|
* @wq: WQ to allocate resources for it from the WQ set |
|
* @wqebb_size: Work Queue Block Byte Size |
|
* @wq_page_size: the page size in the Work Queue |
|
* @q_depth: number of wqebbs in WQ |
|
* @max_wqe_size: maximum WQE size that will be used in the WQ |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
int hinic_wq_allocate(struct hinic_wqs *wqs, struct hinic_wq *wq, |
|
u16 wqebb_size, u32 wq_page_size, u16 q_depth, |
|
u16 max_wqe_size) |
|
{ |
|
struct hinic_hwif *hwif = wqs->hwif; |
|
struct pci_dev *pdev = hwif->pdev; |
|
u16 num_wqebbs_per_page; |
|
u16 wqebb_size_shift; |
|
int err; |
|
|
|
if (!is_power_of_2(wqebb_size)) { |
|
dev_err(&pdev->dev, "wqebb_size must be power of 2\n"); |
|
return -EINVAL; |
|
} |
|
|
|
if (wq_page_size == 0) { |
|
dev_err(&pdev->dev, "wq_page_size must be > 0\n"); |
|
return -EINVAL; |
|
} |
|
|
|
if (q_depth & (q_depth - 1)) { |
|
dev_err(&pdev->dev, "WQ q_depth must be power of 2\n"); |
|
return -EINVAL; |
|
} |
|
|
|
wqebb_size_shift = ilog2(wqebb_size); |
|
num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) |
|
>> wqebb_size_shift; |
|
|
|
if (!is_power_of_2(num_wqebbs_per_page)) { |
|
dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n"); |
|
return -EINVAL; |
|
} |
|
|
|
wq->hwif = hwif; |
|
|
|
err = wqs_next_block(wqs, &wq->page_idx, &wq->block_idx); |
|
if (err) { |
|
dev_err(&pdev->dev, "Failed to get free wqs next block\n"); |
|
return err; |
|
} |
|
|
|
wq->wqebb_size = wqebb_size; |
|
wq->wq_page_size = wq_page_size; |
|
wq->q_depth = q_depth; |
|
wq->max_wqe_size = max_wqe_size; |
|
wq->num_wqebbs_per_page = num_wqebbs_per_page; |
|
wq->wqebbs_per_page_shift = ilog2(num_wqebbs_per_page); |
|
wq->wqebb_size_shift = wqebb_size_shift; |
|
wq->block_vaddr = WQ_BASE_VADDR(wqs, wq); |
|
wq->shadow_block_vaddr = WQ_BASE_ADDR(wqs, wq); |
|
wq->block_paddr = WQ_BASE_PADDR(wqs, wq); |
|
|
|
err = alloc_wq_pages(wq, wqs->hwif, WQ_MAX_PAGES); |
|
if (err) { |
|
dev_err(&pdev->dev, "Failed to allocate wq pages\n"); |
|
goto err_alloc_wq_pages; |
|
} |
|
|
|
atomic_set(&wq->cons_idx, 0); |
|
atomic_set(&wq->prod_idx, 0); |
|
atomic_set(&wq->delta, q_depth); |
|
wq->mask = q_depth - 1; |
|
|
|
return 0; |
|
|
|
err_alloc_wq_pages: |
|
wqs_return_block(wqs, wq->page_idx, wq->block_idx); |
|
return err; |
|
} |
|
|
|
/** |
|
* hinic_wq_free - Free the WQ resources to the WQS |
|
* @wqs: WQ set to free the WQ resources to it |
|
* @wq: WQ to free its resources to the WQ set resources |
|
**/ |
|
void hinic_wq_free(struct hinic_wqs *wqs, struct hinic_wq *wq) |
|
{ |
|
free_wq_pages(wq, wqs->hwif, wq->num_q_pages); |
|
|
|
wqs_return_block(wqs, wq->page_idx, wq->block_idx); |
|
} |
|
|
|
/** |
|
* hinic_wqs_cmdq_alloc - Allocate wqs for cmdqs |
|
* @cmdq_pages: will hold the pages of the cmdq |
|
* @wq: returned wqs |
|
* @hwif: HW interface |
|
* @cmdq_blocks: number of cmdq blocks/wq to allocate |
|
* @wqebb_size: Work Queue Block Byte Size |
|
* @wq_page_size: the page size in the Work Queue |
|
* @q_depth: number of wqebbs in WQ |
|
* @max_wqe_size: maximum WQE size that will be used in the WQ |
|
* |
|
* Return 0 - Success, negative - Failure |
|
**/ |
|
int hinic_wqs_cmdq_alloc(struct hinic_cmdq_pages *cmdq_pages, |
|
struct hinic_wq *wq, struct hinic_hwif *hwif, |
|
int cmdq_blocks, u16 wqebb_size, u32 wq_page_size, |
|
u16 q_depth, u16 max_wqe_size) |
|
{ |
|
struct pci_dev *pdev = hwif->pdev; |
|
u16 num_wqebbs_per_page_shift; |
|
u16 num_wqebbs_per_page; |
|
u16 wqebb_size_shift; |
|
int i, j, err = -ENOMEM; |
|
|
|
if (!is_power_of_2(wqebb_size)) { |
|
dev_err(&pdev->dev, "wqebb_size must be power of 2\n"); |
|
return -EINVAL; |
|
} |
|
|
|
if (wq_page_size == 0) { |
|
dev_err(&pdev->dev, "wq_page_size must be > 0\n"); |
|
return -EINVAL; |
|
} |
|
|
|
if (q_depth & (q_depth - 1)) { |
|
dev_err(&pdev->dev, "WQ q_depth must be power of 2\n"); |
|
return -EINVAL; |
|
} |
|
|
|
wqebb_size_shift = ilog2(wqebb_size); |
|
num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) |
|
>> wqebb_size_shift; |
|
|
|
if (!is_power_of_2(num_wqebbs_per_page)) { |
|
dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n"); |
|
return -EINVAL; |
|
} |
|
|
|
cmdq_pages->hwif = hwif; |
|
|
|
err = cmdq_allocate_page(cmdq_pages); |
|
if (err) { |
|
dev_err(&pdev->dev, "Failed to allocate CMDQ page\n"); |
|
return err; |
|
} |
|
num_wqebbs_per_page_shift = ilog2(num_wqebbs_per_page); |
|
|
|
for (i = 0; i < cmdq_blocks; i++) { |
|
wq[i].hwif = hwif; |
|
wq[i].page_idx = 0; |
|
wq[i].block_idx = i; |
|
|
|
wq[i].wqebb_size = wqebb_size; |
|
wq[i].wq_page_size = wq_page_size; |
|
wq[i].q_depth = q_depth; |
|
wq[i].max_wqe_size = max_wqe_size; |
|
wq[i].num_wqebbs_per_page = num_wqebbs_per_page; |
|
wq[i].wqebbs_per_page_shift = num_wqebbs_per_page_shift; |
|
wq[i].wqebb_size_shift = wqebb_size_shift; |
|
wq[i].block_vaddr = CMDQ_BASE_VADDR(cmdq_pages, &wq[i]); |
|
wq[i].shadow_block_vaddr = CMDQ_BASE_ADDR(cmdq_pages, &wq[i]); |
|
wq[i].block_paddr = CMDQ_BASE_PADDR(cmdq_pages, &wq[i]); |
|
|
|
err = alloc_wq_pages(&wq[i], cmdq_pages->hwif, |
|
CMDQ_WQ_MAX_PAGES); |
|
if (err) { |
|
dev_err(&pdev->dev, "Failed to alloc CMDQ blocks\n"); |
|
goto err_cmdq_block; |
|
} |
|
|
|
atomic_set(&wq[i].cons_idx, 0); |
|
atomic_set(&wq[i].prod_idx, 0); |
|
atomic_set(&wq[i].delta, q_depth); |
|
wq[i].mask = q_depth - 1; |
|
} |
|
|
|
return 0; |
|
|
|
err_cmdq_block: |
|
for (j = 0; j < i; j++) |
|
free_wq_pages(&wq[j], cmdq_pages->hwif, wq[j].num_q_pages); |
|
|
|
cmdq_free_page(cmdq_pages); |
|
return err; |
|
} |
|
|
|
/** |
|
* hinic_wqs_cmdq_free - Free wqs from cmdqs |
|
* @cmdq_pages: hold the pages of the cmdq |
|
* @wq: wqs to free |
|
* @cmdq_blocks: number of wqs to free |
|
**/ |
|
void hinic_wqs_cmdq_free(struct hinic_cmdq_pages *cmdq_pages, |
|
struct hinic_wq *wq, int cmdq_blocks) |
|
{ |
|
int i; |
|
|
|
for (i = 0; i < cmdq_blocks; i++) |
|
free_wq_pages(&wq[i], cmdq_pages->hwif, wq[i].num_q_pages); |
|
|
|
cmdq_free_page(cmdq_pages); |
|
} |
|
|
|
static void copy_wqe_to_shadow(struct hinic_wq *wq, void *shadow_addr, |
|
int num_wqebbs, u16 idx) |
|
{ |
|
void *wqebb_addr; |
|
int i; |
|
|
|
for (i = 0; i < num_wqebbs; i++, idx++) { |
|
idx = MASKED_WQE_IDX(wq, idx); |
|
wqebb_addr = WQ_PAGE_ADDR(wq, idx) + |
|
WQE_PAGE_OFF(wq, idx); |
|
|
|
memcpy(shadow_addr, wqebb_addr, wq->wqebb_size); |
|
|
|
shadow_addr += wq->wqebb_size; |
|
} |
|
} |
|
|
|
static void copy_wqe_from_shadow(struct hinic_wq *wq, void *shadow_addr, |
|
int num_wqebbs, u16 idx) |
|
{ |
|
void *wqebb_addr; |
|
int i; |
|
|
|
for (i = 0; i < num_wqebbs; i++, idx++) { |
|
idx = MASKED_WQE_IDX(wq, idx); |
|
wqebb_addr = WQ_PAGE_ADDR(wq, idx) + |
|
WQE_PAGE_OFF(wq, idx); |
|
|
|
memcpy(wqebb_addr, shadow_addr, wq->wqebb_size); |
|
shadow_addr += wq->wqebb_size; |
|
} |
|
} |
|
|
|
/** |
|
* hinic_get_wqe - get wqe ptr in the current pi and update the pi |
|
* @wq: wq to get wqe from |
|
* @wqe_size: wqe size |
|
* @prod_idx: returned pi |
|
* |
|
* Return wqe pointer |
|
**/ |
|
struct hinic_hw_wqe *hinic_get_wqe(struct hinic_wq *wq, unsigned int wqe_size, |
|
u16 *prod_idx) |
|
{ |
|
int curr_pg, end_pg, num_wqebbs; |
|
u16 curr_prod_idx, end_prod_idx; |
|
|
|
*prod_idx = MASKED_WQE_IDX(wq, atomic_read(&wq->prod_idx)); |
|
|
|
num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) >> wq->wqebb_size_shift; |
|
|
|
if (atomic_sub_return(num_wqebbs, &wq->delta) <= 0) { |
|
atomic_add(num_wqebbs, &wq->delta); |
|
return ERR_PTR(-EBUSY); |
|
} |
|
|
|
end_prod_idx = atomic_add_return(num_wqebbs, &wq->prod_idx); |
|
|
|
end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx); |
|
curr_prod_idx = end_prod_idx - num_wqebbs; |
|
curr_prod_idx = MASKED_WQE_IDX(wq, curr_prod_idx); |
|
|
|
/* end prod index points to the next wqebb, therefore minus 1 */ |
|
end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx - 1); |
|
|
|
curr_pg = WQE_PAGE_NUM(wq, curr_prod_idx); |
|
end_pg = WQE_PAGE_NUM(wq, end_prod_idx); |
|
|
|
*prod_idx = curr_prod_idx; |
|
|
|
/* If we only have one page, still need to get shadown wqe when |
|
* wqe rolling-over page |
|
*/ |
|
if (curr_pg != end_pg || MASKED_WQE_IDX(wq, end_prod_idx) < *prod_idx) { |
|
void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; |
|
|
|
copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *prod_idx); |
|
|
|
wq->shadow_idx[curr_pg] = *prod_idx; |
|
return shadow_addr; |
|
} |
|
|
|
return WQ_PAGE_ADDR(wq, *prod_idx) + WQE_PAGE_OFF(wq, *prod_idx); |
|
} |
|
|
|
/** |
|
* hinic_return_wqe - return the wqe when transmit failed |
|
* @wq: wq to return wqe |
|
* @wqe_size: wqe size |
|
**/ |
|
void hinic_return_wqe(struct hinic_wq *wq, unsigned int wqe_size) |
|
{ |
|
int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; |
|
|
|
atomic_sub(num_wqebbs, &wq->prod_idx); |
|
|
|
atomic_add(num_wqebbs, &wq->delta); |
|
} |
|
|
|
/** |
|
* hinic_put_wqe - return the wqe place to use for a new wqe |
|
* @wq: wq to return wqe |
|
* @wqe_size: wqe size |
|
**/ |
|
void hinic_put_wqe(struct hinic_wq *wq, unsigned int wqe_size) |
|
{ |
|
int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) |
|
>> wq->wqebb_size_shift; |
|
|
|
atomic_add(num_wqebbs, &wq->cons_idx); |
|
|
|
atomic_add(num_wqebbs, &wq->delta); |
|
} |
|
|
|
/** |
|
* hinic_read_wqe - read wqe ptr in the current ci |
|
* @wq: wq to get read from |
|
* @wqe_size: wqe size |
|
* @cons_idx: returned ci |
|
* |
|
* Return wqe pointer |
|
**/ |
|
struct hinic_hw_wqe *hinic_read_wqe(struct hinic_wq *wq, unsigned int wqe_size, |
|
u16 *cons_idx) |
|
{ |
|
int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) |
|
>> wq->wqebb_size_shift; |
|
u16 curr_cons_idx, end_cons_idx; |
|
int curr_pg, end_pg; |
|
|
|
if ((atomic_read(&wq->delta) + num_wqebbs) > wq->q_depth) |
|
return ERR_PTR(-EBUSY); |
|
|
|
curr_cons_idx = atomic_read(&wq->cons_idx); |
|
|
|
curr_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx); |
|
end_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx + num_wqebbs - 1); |
|
|
|
curr_pg = WQE_PAGE_NUM(wq, curr_cons_idx); |
|
end_pg = WQE_PAGE_NUM(wq, end_cons_idx); |
|
|
|
*cons_idx = curr_cons_idx; |
|
|
|
if (curr_pg != end_pg) { |
|
void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; |
|
|
|
copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *cons_idx); |
|
return shadow_addr; |
|
} |
|
|
|
return WQ_PAGE_ADDR(wq, *cons_idx) + WQE_PAGE_OFF(wq, *cons_idx); |
|
} |
|
|
|
/** |
|
* hinic_read_wqe_direct - read wqe directly from ci position |
|
* @wq: wq |
|
* @cons_idx: ci position |
|
* |
|
* Return wqe |
|
**/ |
|
struct hinic_hw_wqe *hinic_read_wqe_direct(struct hinic_wq *wq, u16 cons_idx) |
|
{ |
|
return WQ_PAGE_ADDR(wq, cons_idx) + WQE_PAGE_OFF(wq, cons_idx); |
|
} |
|
|
|
/** |
|
* wqe_shadow - check if a wqe is shadow |
|
* @wq: wq of the wqe |
|
* @wqe: the wqe for shadow checking |
|
* |
|
* Return true - shadow, false - Not shadow |
|
**/ |
|
static inline bool wqe_shadow(struct hinic_wq *wq, struct hinic_hw_wqe *wqe) |
|
{ |
|
size_t wqe_shadow_size = wq->num_q_pages * wq->max_wqe_size; |
|
|
|
return WQE_IN_RANGE(wqe, wq->shadow_wqe, |
|
&wq->shadow_wqe[wqe_shadow_size]); |
|
} |
|
|
|
/** |
|
* hinic_write_wqe - write the wqe to the wq |
|
* @wq: wq to write wqe to |
|
* @wqe: wqe to write |
|
* @wqe_size: wqe size |
|
**/ |
|
void hinic_write_wqe(struct hinic_wq *wq, struct hinic_hw_wqe *wqe, |
|
unsigned int wqe_size) |
|
{ |
|
int curr_pg, num_wqebbs; |
|
void *shadow_addr; |
|
u16 prod_idx; |
|
|
|
if (wqe_shadow(wq, wqe)) { |
|
curr_pg = WQE_SHADOW_PAGE(wq, wqe); |
|
|
|
prod_idx = wq->shadow_idx[curr_pg]; |
|
num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; |
|
shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; |
|
|
|
copy_wqe_from_shadow(wq, shadow_addr, num_wqebbs, prod_idx); |
|
} |
|
}
|
|
|