forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
569 lines
12 KiB
569 lines
12 KiB
// SPDX-License-Identifier: GPL-2.0+ |
|
// Copyright 2019 IBM Corp. |
|
#include <linux/idr.h> |
|
#include "ocxl_internal.h" |
|
|
|
static struct ocxl_fn *ocxl_fn_get(struct ocxl_fn *fn) |
|
{ |
|
return (get_device(&fn->dev) == NULL) ? NULL : fn; |
|
} |
|
|
|
static void ocxl_fn_put(struct ocxl_fn *fn) |
|
{ |
|
put_device(&fn->dev); |
|
} |
|
|
|
static struct ocxl_afu *alloc_afu(struct ocxl_fn *fn) |
|
{ |
|
struct ocxl_afu *afu; |
|
|
|
afu = kzalloc(sizeof(struct ocxl_afu), GFP_KERNEL); |
|
if (!afu) |
|
return NULL; |
|
|
|
kref_init(&afu->kref); |
|
mutex_init(&afu->contexts_lock); |
|
mutex_init(&afu->afu_control_lock); |
|
idr_init(&afu->contexts_idr); |
|
afu->fn = fn; |
|
ocxl_fn_get(fn); |
|
return afu; |
|
} |
|
|
|
static void free_afu(struct kref *kref) |
|
{ |
|
struct ocxl_afu *afu = container_of(kref, struct ocxl_afu, kref); |
|
|
|
idr_destroy(&afu->contexts_idr); |
|
ocxl_fn_put(afu->fn); |
|
kfree(afu); |
|
} |
|
|
|
void ocxl_afu_get(struct ocxl_afu *afu) |
|
{ |
|
kref_get(&afu->kref); |
|
} |
|
EXPORT_SYMBOL_GPL(ocxl_afu_get); |
|
|
|
void ocxl_afu_put(struct ocxl_afu *afu) |
|
{ |
|
kref_put(&afu->kref, free_afu); |
|
} |
|
EXPORT_SYMBOL_GPL(ocxl_afu_put); |
|
|
|
static int assign_afu_actag(struct ocxl_afu *afu) |
|
{ |
|
struct ocxl_fn *fn = afu->fn; |
|
int actag_count, actag_offset; |
|
struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent); |
|
|
|
/* |
|
* if there were not enough actags for the function, each afu |
|
* reduces its count as well |
|
*/ |
|
actag_count = afu->config.actag_supported * |
|
fn->actag_enabled / fn->actag_supported; |
|
actag_offset = ocxl_actag_afu_alloc(fn, actag_count); |
|
if (actag_offset < 0) { |
|
dev_err(&pci_dev->dev, "Can't allocate %d actags for AFU: %d\n", |
|
actag_count, actag_offset); |
|
return actag_offset; |
|
} |
|
afu->actag_base = fn->actag_base + actag_offset; |
|
afu->actag_enabled = actag_count; |
|
|
|
ocxl_config_set_afu_actag(pci_dev, afu->config.dvsec_afu_control_pos, |
|
afu->actag_base, afu->actag_enabled); |
|
dev_dbg(&pci_dev->dev, "actag base=%d enabled=%d\n", |
|
afu->actag_base, afu->actag_enabled); |
|
return 0; |
|
} |
|
|
|
static void reclaim_afu_actag(struct ocxl_afu *afu) |
|
{ |
|
struct ocxl_fn *fn = afu->fn; |
|
int start_offset, size; |
|
|
|
start_offset = afu->actag_base - fn->actag_base; |
|
size = afu->actag_enabled; |
|
ocxl_actag_afu_free(afu->fn, start_offset, size); |
|
} |
|
|
|
static int assign_afu_pasid(struct ocxl_afu *afu) |
|
{ |
|
struct ocxl_fn *fn = afu->fn; |
|
int pasid_count, pasid_offset; |
|
struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent); |
|
|
|
/* |
|
* We only support the case where the function configuration |
|
* requested enough PASIDs to cover all AFUs. |
|
*/ |
|
pasid_count = 1 << afu->config.pasid_supported_log; |
|
pasid_offset = ocxl_pasid_afu_alloc(fn, pasid_count); |
|
if (pasid_offset < 0) { |
|
dev_err(&pci_dev->dev, "Can't allocate %d PASIDs for AFU: %d\n", |
|
pasid_count, pasid_offset); |
|
return pasid_offset; |
|
} |
|
afu->pasid_base = fn->pasid_base + pasid_offset; |
|
afu->pasid_count = 0; |
|
afu->pasid_max = pasid_count; |
|
|
|
ocxl_config_set_afu_pasid(pci_dev, afu->config.dvsec_afu_control_pos, |
|
afu->pasid_base, |
|
afu->config.pasid_supported_log); |
|
dev_dbg(&pci_dev->dev, "PASID base=%d, enabled=%d\n", |
|
afu->pasid_base, pasid_count); |
|
return 0; |
|
} |
|
|
|
static void reclaim_afu_pasid(struct ocxl_afu *afu) |
|
{ |
|
struct ocxl_fn *fn = afu->fn; |
|
int start_offset, size; |
|
|
|
start_offset = afu->pasid_base - fn->pasid_base; |
|
size = 1 << afu->config.pasid_supported_log; |
|
ocxl_pasid_afu_free(afu->fn, start_offset, size); |
|
} |
|
|
|
static int reserve_fn_bar(struct ocxl_fn *fn, int bar) |
|
{ |
|
struct pci_dev *dev = to_pci_dev(fn->dev.parent); |
|
int rc, idx; |
|
|
|
if (bar != 0 && bar != 2 && bar != 4) |
|
return -EINVAL; |
|
|
|
idx = bar >> 1; |
|
if (fn->bar_used[idx]++ == 0) { |
|
rc = pci_request_region(dev, bar, "ocxl"); |
|
if (rc) |
|
return rc; |
|
} |
|
return 0; |
|
} |
|
|
|
static void release_fn_bar(struct ocxl_fn *fn, int bar) |
|
{ |
|
struct pci_dev *dev = to_pci_dev(fn->dev.parent); |
|
int idx; |
|
|
|
if (bar != 0 && bar != 2 && bar != 4) |
|
return; |
|
|
|
idx = bar >> 1; |
|
if (--fn->bar_used[idx] == 0) |
|
pci_release_region(dev, bar); |
|
WARN_ON(fn->bar_used[idx] < 0); |
|
} |
|
|
|
static int map_mmio_areas(struct ocxl_afu *afu) |
|
{ |
|
int rc; |
|
struct pci_dev *pci_dev = to_pci_dev(afu->fn->dev.parent); |
|
|
|
rc = reserve_fn_bar(afu->fn, afu->config.global_mmio_bar); |
|
if (rc) |
|
return rc; |
|
|
|
rc = reserve_fn_bar(afu->fn, afu->config.pp_mmio_bar); |
|
if (rc) { |
|
release_fn_bar(afu->fn, afu->config.global_mmio_bar); |
|
return rc; |
|
} |
|
|
|
afu->global_mmio_start = |
|
pci_resource_start(pci_dev, afu->config.global_mmio_bar) + |
|
afu->config.global_mmio_offset; |
|
afu->pp_mmio_start = |
|
pci_resource_start(pci_dev, afu->config.pp_mmio_bar) + |
|
afu->config.pp_mmio_offset; |
|
|
|
afu->global_mmio_ptr = ioremap(afu->global_mmio_start, |
|
afu->config.global_mmio_size); |
|
if (!afu->global_mmio_ptr) { |
|
release_fn_bar(afu->fn, afu->config.pp_mmio_bar); |
|
release_fn_bar(afu->fn, afu->config.global_mmio_bar); |
|
dev_err(&pci_dev->dev, "Error mapping global mmio area\n"); |
|
return -ENOMEM; |
|
} |
|
|
|
/* |
|
* Leave an empty page between the per-process mmio area and |
|
* the AFU interrupt mappings |
|
*/ |
|
afu->irq_base_offset = afu->config.pp_mmio_stride + PAGE_SIZE; |
|
return 0; |
|
} |
|
|
|
static void unmap_mmio_areas(struct ocxl_afu *afu) |
|
{ |
|
if (afu->global_mmio_ptr) { |
|
iounmap(afu->global_mmio_ptr); |
|
afu->global_mmio_ptr = NULL; |
|
} |
|
afu->global_mmio_start = 0; |
|
afu->pp_mmio_start = 0; |
|
release_fn_bar(afu->fn, afu->config.pp_mmio_bar); |
|
release_fn_bar(afu->fn, afu->config.global_mmio_bar); |
|
} |
|
|
|
static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev) |
|
{ |
|
int rc; |
|
|
|
rc = ocxl_config_read_afu(dev, &afu->fn->config, &afu->config, afu_idx); |
|
if (rc) |
|
return rc; |
|
|
|
rc = assign_afu_actag(afu); |
|
if (rc) |
|
return rc; |
|
|
|
rc = assign_afu_pasid(afu); |
|
if (rc) |
|
goto err_free_actag; |
|
|
|
rc = map_mmio_areas(afu); |
|
if (rc) |
|
goto err_free_pasid; |
|
|
|
return 0; |
|
|
|
err_free_pasid: |
|
reclaim_afu_pasid(afu); |
|
err_free_actag: |
|
reclaim_afu_actag(afu); |
|
return rc; |
|
} |
|
|
|
static void deconfigure_afu(struct ocxl_afu *afu) |
|
{ |
|
unmap_mmio_areas(afu); |
|
reclaim_afu_pasid(afu); |
|
reclaim_afu_actag(afu); |
|
} |
|
|
|
static int activate_afu(struct pci_dev *dev, struct ocxl_afu *afu) |
|
{ |
|
ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 1); |
|
|
|
return 0; |
|
} |
|
|
|
static void deactivate_afu(struct ocxl_afu *afu) |
|
{ |
|
struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent); |
|
|
|
ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 0); |
|
} |
|
|
|
static int init_afu(struct pci_dev *dev, struct ocxl_fn *fn, u8 afu_idx) |
|
{ |
|
int rc; |
|
struct ocxl_afu *afu; |
|
|
|
afu = alloc_afu(fn); |
|
if (!afu) |
|
return -ENOMEM; |
|
|
|
rc = configure_afu(afu, afu_idx, dev); |
|
if (rc) { |
|
ocxl_afu_put(afu); |
|
return rc; |
|
} |
|
|
|
rc = activate_afu(dev, afu); |
|
if (rc) { |
|
deconfigure_afu(afu); |
|
ocxl_afu_put(afu); |
|
return rc; |
|
} |
|
|
|
list_add_tail(&afu->list, &fn->afu_list); |
|
|
|
return 0; |
|
} |
|
|
|
static void remove_afu(struct ocxl_afu *afu) |
|
{ |
|
list_del(&afu->list); |
|
ocxl_context_detach_all(afu); |
|
deactivate_afu(afu); |
|
deconfigure_afu(afu); |
|
ocxl_afu_put(afu); // matches the implicit get in alloc_afu |
|
} |
|
|
|
static struct ocxl_fn *alloc_function(void) |
|
{ |
|
struct ocxl_fn *fn; |
|
|
|
fn = kzalloc(sizeof(struct ocxl_fn), GFP_KERNEL); |
|
if (!fn) |
|
return NULL; |
|
|
|
INIT_LIST_HEAD(&fn->afu_list); |
|
INIT_LIST_HEAD(&fn->pasid_list); |
|
INIT_LIST_HEAD(&fn->actag_list); |
|
|
|
return fn; |
|
} |
|
|
|
static void free_function(struct ocxl_fn *fn) |
|
{ |
|
WARN_ON(!list_empty(&fn->afu_list)); |
|
WARN_ON(!list_empty(&fn->pasid_list)); |
|
kfree(fn); |
|
} |
|
|
|
static void free_function_dev(struct device *dev) |
|
{ |
|
struct ocxl_fn *fn = container_of(dev, struct ocxl_fn, dev); |
|
|
|
free_function(fn); |
|
} |
|
|
|
static int set_function_device(struct ocxl_fn *fn, struct pci_dev *dev) |
|
{ |
|
fn->dev.parent = &dev->dev; |
|
fn->dev.release = free_function_dev; |
|
return dev_set_name(&fn->dev, "ocxlfn.%s", dev_name(&dev->dev)); |
|
} |
|
|
|
static int assign_function_actag(struct ocxl_fn *fn) |
|
{ |
|
struct pci_dev *dev = to_pci_dev(fn->dev.parent); |
|
u16 base, enabled, supported; |
|
int rc; |
|
|
|
rc = ocxl_config_get_actag_info(dev, &base, &enabled, &supported); |
|
if (rc) |
|
return rc; |
|
|
|
fn->actag_base = base; |
|
fn->actag_enabled = enabled; |
|
fn->actag_supported = supported; |
|
|
|
ocxl_config_set_actag(dev, fn->config.dvsec_function_pos, |
|
fn->actag_base, fn->actag_enabled); |
|
dev_dbg(&fn->dev, "actag range starting at %d, enabled %d\n", |
|
fn->actag_base, fn->actag_enabled); |
|
return 0; |
|
} |
|
|
|
static int set_function_pasid(struct ocxl_fn *fn) |
|
{ |
|
struct pci_dev *dev = to_pci_dev(fn->dev.parent); |
|
int rc, desired_count, max_count; |
|
|
|
/* A function may not require any PASID */ |
|
if (fn->config.max_pasid_log < 0) |
|
return 0; |
|
|
|
rc = ocxl_config_get_pasid_info(dev, &max_count); |
|
if (rc) |
|
return rc; |
|
|
|
desired_count = 1 << fn->config.max_pasid_log; |
|
|
|
if (desired_count > max_count) { |
|
dev_err(&fn->dev, |
|
"Function requires more PASIDs than is available (%d vs. %d)\n", |
|
desired_count, max_count); |
|
return -ENOSPC; |
|
} |
|
|
|
fn->pasid_base = 0; |
|
return 0; |
|
} |
|
|
|
static int configure_function(struct ocxl_fn *fn, struct pci_dev *dev) |
|
{ |
|
int rc; |
|
|
|
rc = pci_enable_device(dev); |
|
if (rc) { |
|
dev_err(&dev->dev, "pci_enable_device failed: %d\n", rc); |
|
return rc; |
|
} |
|
|
|
/* |
|
* Once it has been confirmed to work on our hardware, we |
|
* should reset the function, to force the adapter to restart |
|
* from scratch. |
|
* A function reset would also reset all its AFUs. |
|
* |
|
* Some hints for implementation: |
|
* |
|
* - there's not status bit to know when the reset is done. We |
|
* should try reading the config space to know when it's |
|
* done. |
|
* - probably something like: |
|
* Reset |
|
* wait 100ms |
|
* issue config read |
|
* allow device up to 1 sec to return success on config |
|
* read before declaring it broken |
|
* |
|
* Some shared logic on the card (CFG, TLX) won't be reset, so |
|
* there's no guarantee that it will be enough. |
|
*/ |
|
rc = ocxl_config_read_function(dev, &fn->config); |
|
if (rc) |
|
return rc; |
|
|
|
rc = set_function_device(fn, dev); |
|
if (rc) |
|
return rc; |
|
|
|
rc = assign_function_actag(fn); |
|
if (rc) |
|
return rc; |
|
|
|
rc = set_function_pasid(fn); |
|
if (rc) |
|
return rc; |
|
|
|
rc = ocxl_link_setup(dev, 0, &fn->link); |
|
if (rc) |
|
return rc; |
|
|
|
rc = ocxl_config_set_TL(dev, fn->config.dvsec_tl_pos); |
|
if (rc) { |
|
ocxl_link_release(dev, fn->link); |
|
return rc; |
|
} |
|
return 0; |
|
} |
|
|
|
static void deconfigure_function(struct ocxl_fn *fn) |
|
{ |
|
struct pci_dev *dev = to_pci_dev(fn->dev.parent); |
|
|
|
ocxl_link_release(dev, fn->link); |
|
pci_disable_device(dev); |
|
} |
|
|
|
static struct ocxl_fn *init_function(struct pci_dev *dev) |
|
{ |
|
struct ocxl_fn *fn; |
|
int rc; |
|
|
|
fn = alloc_function(); |
|
if (!fn) |
|
return ERR_PTR(-ENOMEM); |
|
|
|
rc = configure_function(fn, dev); |
|
if (rc) { |
|
free_function(fn); |
|
return ERR_PTR(rc); |
|
} |
|
|
|
rc = device_register(&fn->dev); |
|
if (rc) { |
|
deconfigure_function(fn); |
|
put_device(&fn->dev); |
|
return ERR_PTR(rc); |
|
} |
|
return fn; |
|
} |
|
|
|
// Device detection & initialisation |
|
|
|
struct ocxl_fn *ocxl_function_open(struct pci_dev *dev) |
|
{ |
|
int rc, afu_count = 0; |
|
u8 afu; |
|
struct ocxl_fn *fn; |
|
|
|
if (!radix_enabled()) { |
|
dev_err(&dev->dev, "Unsupported memory model (hash)\n"); |
|
return ERR_PTR(-ENODEV); |
|
} |
|
|
|
fn = init_function(dev); |
|
if (IS_ERR(fn)) { |
|
dev_err(&dev->dev, "function init failed: %li\n", |
|
PTR_ERR(fn)); |
|
return fn; |
|
} |
|
|
|
for (afu = 0; afu <= fn->config.max_afu_index; afu++) { |
|
rc = ocxl_config_check_afu_index(dev, &fn->config, afu); |
|
if (rc > 0) { |
|
rc = init_afu(dev, fn, afu); |
|
if (rc) { |
|
dev_err(&dev->dev, |
|
"Can't initialize AFU index %d\n", afu); |
|
continue; |
|
} |
|
afu_count++; |
|
} |
|
} |
|
dev_info(&dev->dev, "%d AFU(s) configured\n", afu_count); |
|
return fn; |
|
} |
|
EXPORT_SYMBOL_GPL(ocxl_function_open); |
|
|
|
struct list_head *ocxl_function_afu_list(struct ocxl_fn *fn) |
|
{ |
|
return &fn->afu_list; |
|
} |
|
EXPORT_SYMBOL_GPL(ocxl_function_afu_list); |
|
|
|
struct ocxl_afu *ocxl_function_fetch_afu(struct ocxl_fn *fn, u8 afu_idx) |
|
{ |
|
struct ocxl_afu *afu; |
|
|
|
list_for_each_entry(afu, &fn->afu_list, list) { |
|
if (afu->config.idx == afu_idx) |
|
return afu; |
|
} |
|
|
|
return NULL; |
|
} |
|
EXPORT_SYMBOL_GPL(ocxl_function_fetch_afu); |
|
|
|
const struct ocxl_fn_config *ocxl_function_config(struct ocxl_fn *fn) |
|
{ |
|
return &fn->config; |
|
} |
|
EXPORT_SYMBOL_GPL(ocxl_function_config); |
|
|
|
void ocxl_function_close(struct ocxl_fn *fn) |
|
{ |
|
struct ocxl_afu *afu, *tmp; |
|
|
|
list_for_each_entry_safe(afu, tmp, &fn->afu_list, list) { |
|
remove_afu(afu); |
|
} |
|
|
|
deconfigure_function(fn); |
|
device_unregister(&fn->dev); |
|
} |
|
EXPORT_SYMBOL_GPL(ocxl_function_close); |
|
|
|
// AFU Metadata |
|
|
|
struct ocxl_afu_config *ocxl_afu_config(struct ocxl_afu *afu) |
|
{ |
|
return &afu->config; |
|
} |
|
EXPORT_SYMBOL_GPL(ocxl_afu_config); |
|
|
|
void ocxl_afu_set_private(struct ocxl_afu *afu, void *private) |
|
{ |
|
afu->private = private; |
|
} |
|
EXPORT_SYMBOL_GPL(ocxl_afu_set_private); |
|
|
|
void *ocxl_afu_get_private(struct ocxl_afu *afu) |
|
{ |
|
if (afu) |
|
return afu->private; |
|
|
|
return NULL; |
|
} |
|
EXPORT_SYMBOL_GPL(ocxl_afu_get_private);
|
|
|