mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
513 lines
12 KiB
513 lines
12 KiB
// SPDX-License-Identifier: GPL-2.0 |
|
/* |
|
* VFIO based Physical Subchannel device driver |
|
* |
|
* Copyright IBM Corp. 2017 |
|
* Copyright Red Hat, Inc. 2019 |
|
* |
|
* Author(s): Dong Jia Shi <[email protected]> |
|
* Xiao Feng Ren <[email protected]> |
|
* Cornelia Huck <[email protected]> |
|
*/ |
|
|
|
#include <linux/module.h> |
|
#include <linux/init.h> |
|
#include <linux/device.h> |
|
#include <linux/slab.h> |
|
#include <linux/uuid.h> |
|
#include <linux/mdev.h> |
|
|
|
#include <asm/isc.h> |
|
|
|
#include "chp.h" |
|
#include "ioasm.h" |
|
#include "css.h" |
|
#include "vfio_ccw_private.h" |
|
|
|
struct workqueue_struct *vfio_ccw_work_q; |
|
static struct kmem_cache *vfio_ccw_io_region; |
|
static struct kmem_cache *vfio_ccw_cmd_region; |
|
static struct kmem_cache *vfio_ccw_schib_region; |
|
static struct kmem_cache *vfio_ccw_crw_region; |
|
|
|
debug_info_t *vfio_ccw_debug_msg_id; |
|
debug_info_t *vfio_ccw_debug_trace_id; |
|
|
|
/* |
|
* Helpers |
|
*/ |
|
int vfio_ccw_sch_quiesce(struct subchannel *sch) |
|
{ |
|
struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); |
|
DECLARE_COMPLETION_ONSTACK(completion); |
|
int iretry, ret = 0; |
|
|
|
spin_lock_irq(sch->lock); |
|
if (!sch->schib.pmcw.ena) |
|
goto out_unlock; |
|
ret = cio_disable_subchannel(sch); |
|
if (ret != -EBUSY) |
|
goto out_unlock; |
|
|
|
iretry = 255; |
|
do { |
|
|
|
ret = cio_cancel_halt_clear(sch, &iretry); |
|
|
|
if (ret == -EIO) { |
|
pr_err("vfio_ccw: could not quiesce subchannel 0.%x.%04x!\n", |
|
sch->schid.ssid, sch->schid.sch_no); |
|
break; |
|
} |
|
|
|
/* |
|
* Flush all I/O and wait for |
|
* cancel/halt/clear completion. |
|
*/ |
|
private->completion = &completion; |
|
spin_unlock_irq(sch->lock); |
|
|
|
if (ret == -EBUSY) |
|
wait_for_completion_timeout(&completion, 3*HZ); |
|
|
|
private->completion = NULL; |
|
flush_workqueue(vfio_ccw_work_q); |
|
spin_lock_irq(sch->lock); |
|
ret = cio_disable_subchannel(sch); |
|
} while (ret == -EBUSY); |
|
out_unlock: |
|
private->state = VFIO_CCW_STATE_NOT_OPER; |
|
spin_unlock_irq(sch->lock); |
|
return ret; |
|
} |
|
|
|
static void vfio_ccw_sch_io_todo(struct work_struct *work) |
|
{ |
|
struct vfio_ccw_private *private; |
|
struct irb *irb; |
|
bool is_final; |
|
|
|
private = container_of(work, struct vfio_ccw_private, io_work); |
|
irb = &private->irb; |
|
|
|
is_final = !(scsw_actl(&irb->scsw) & |
|
(SCSW_ACTL_DEVACT | SCSW_ACTL_SCHACT)); |
|
if (scsw_is_solicited(&irb->scsw)) { |
|
cp_update_scsw(&private->cp, &irb->scsw); |
|
if (is_final && private->state == VFIO_CCW_STATE_CP_PENDING) |
|
cp_free(&private->cp); |
|
} |
|
mutex_lock(&private->io_mutex); |
|
memcpy(private->io_region->irb_area, irb, sizeof(*irb)); |
|
mutex_unlock(&private->io_mutex); |
|
|
|
if (private->mdev && is_final) |
|
private->state = VFIO_CCW_STATE_IDLE; |
|
|
|
if (private->io_trigger) |
|
eventfd_signal(private->io_trigger, 1); |
|
} |
|
|
|
static void vfio_ccw_crw_todo(struct work_struct *work) |
|
{ |
|
struct vfio_ccw_private *private; |
|
|
|
private = container_of(work, struct vfio_ccw_private, crw_work); |
|
|
|
if (!list_empty(&private->crw) && private->crw_trigger) |
|
eventfd_signal(private->crw_trigger, 1); |
|
} |
|
|
|
/* |
|
* Css driver callbacks |
|
*/ |
|
static void vfio_ccw_sch_irq(struct subchannel *sch) |
|
{ |
|
struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); |
|
|
|
inc_irq_stat(IRQIO_CIO); |
|
vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_INTERRUPT); |
|
} |
|
|
|
static void vfio_ccw_free_regions(struct vfio_ccw_private *private) |
|
{ |
|
if (private->crw_region) |
|
kmem_cache_free(vfio_ccw_crw_region, private->crw_region); |
|
if (private->schib_region) |
|
kmem_cache_free(vfio_ccw_schib_region, private->schib_region); |
|
if (private->cmd_region) |
|
kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region); |
|
if (private->io_region) |
|
kmem_cache_free(vfio_ccw_io_region, private->io_region); |
|
} |
|
|
|
static int vfio_ccw_sch_probe(struct subchannel *sch) |
|
{ |
|
struct pmcw *pmcw = &sch->schib.pmcw; |
|
struct vfio_ccw_private *private; |
|
int ret = -ENOMEM; |
|
|
|
if (pmcw->qf) { |
|
dev_warn(&sch->dev, "vfio: ccw: does not support QDIO: %s\n", |
|
dev_name(&sch->dev)); |
|
return -ENODEV; |
|
} |
|
|
|
private = kzalloc(sizeof(*private), GFP_KERNEL | GFP_DMA); |
|
if (!private) |
|
return -ENOMEM; |
|
|
|
private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1), |
|
GFP_KERNEL); |
|
if (!private->cp.guest_cp) |
|
goto out_free; |
|
|
|
private->io_region = kmem_cache_zalloc(vfio_ccw_io_region, |
|
GFP_KERNEL | GFP_DMA); |
|
if (!private->io_region) |
|
goto out_free; |
|
|
|
private->cmd_region = kmem_cache_zalloc(vfio_ccw_cmd_region, |
|
GFP_KERNEL | GFP_DMA); |
|
if (!private->cmd_region) |
|
goto out_free; |
|
|
|
private->schib_region = kmem_cache_zalloc(vfio_ccw_schib_region, |
|
GFP_KERNEL | GFP_DMA); |
|
|
|
if (!private->schib_region) |
|
goto out_free; |
|
|
|
private->crw_region = kmem_cache_zalloc(vfio_ccw_crw_region, |
|
GFP_KERNEL | GFP_DMA); |
|
|
|
if (!private->crw_region) |
|
goto out_free; |
|
|
|
private->sch = sch; |
|
dev_set_drvdata(&sch->dev, private); |
|
mutex_init(&private->io_mutex); |
|
|
|
spin_lock_irq(sch->lock); |
|
private->state = VFIO_CCW_STATE_NOT_OPER; |
|
sch->isc = VFIO_CCW_ISC; |
|
ret = cio_enable_subchannel(sch, (u32)(unsigned long)sch); |
|
spin_unlock_irq(sch->lock); |
|
if (ret) |
|
goto out_free; |
|
|
|
INIT_LIST_HEAD(&private->crw); |
|
INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo); |
|
INIT_WORK(&private->crw_work, vfio_ccw_crw_todo); |
|
atomic_set(&private->avail, 1); |
|
private->state = VFIO_CCW_STATE_STANDBY; |
|
|
|
ret = vfio_ccw_mdev_reg(sch); |
|
if (ret) |
|
goto out_disable; |
|
|
|
if (dev_get_uevent_suppress(&sch->dev)) { |
|
dev_set_uevent_suppress(&sch->dev, 0); |
|
kobject_uevent(&sch->dev.kobj, KOBJ_ADD); |
|
} |
|
|
|
VFIO_CCW_MSG_EVENT(4, "bound to subchannel %x.%x.%04x\n", |
|
sch->schid.cssid, sch->schid.ssid, |
|
sch->schid.sch_no); |
|
return 0; |
|
|
|
out_disable: |
|
cio_disable_subchannel(sch); |
|
out_free: |
|
dev_set_drvdata(&sch->dev, NULL); |
|
vfio_ccw_free_regions(private); |
|
kfree(private->cp.guest_cp); |
|
kfree(private); |
|
return ret; |
|
} |
|
|
|
static int vfio_ccw_sch_remove(struct subchannel *sch) |
|
{ |
|
struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); |
|
struct vfio_ccw_crw *crw, *temp; |
|
|
|
vfio_ccw_sch_quiesce(sch); |
|
|
|
list_for_each_entry_safe(crw, temp, &private->crw, next) { |
|
list_del(&crw->next); |
|
kfree(crw); |
|
} |
|
|
|
vfio_ccw_mdev_unreg(sch); |
|
|
|
dev_set_drvdata(&sch->dev, NULL); |
|
|
|
vfio_ccw_free_regions(private); |
|
kfree(private->cp.guest_cp); |
|
kfree(private); |
|
|
|
VFIO_CCW_MSG_EVENT(4, "unbound from subchannel %x.%x.%04x\n", |
|
sch->schid.cssid, sch->schid.ssid, |
|
sch->schid.sch_no); |
|
return 0; |
|
} |
|
|
|
static void vfio_ccw_sch_shutdown(struct subchannel *sch) |
|
{ |
|
vfio_ccw_sch_quiesce(sch); |
|
} |
|
|
|
/** |
|
* vfio_ccw_sch_event - process subchannel event |
|
* @sch: subchannel |
|
* @process: non-zero if function is called in process context |
|
* |
|
* An unspecified event occurred for this subchannel. Adjust data according |
|
* to the current operational state of the subchannel. Return zero when the |
|
* event has been handled sufficiently or -EAGAIN when this function should |
|
* be called again in process context. |
|
*/ |
|
static int vfio_ccw_sch_event(struct subchannel *sch, int process) |
|
{ |
|
struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); |
|
unsigned long flags; |
|
int rc = -EAGAIN; |
|
|
|
spin_lock_irqsave(sch->lock, flags); |
|
if (!device_is_registered(&sch->dev)) |
|
goto out_unlock; |
|
|
|
if (work_pending(&sch->todo_work)) |
|
goto out_unlock; |
|
|
|
if (cio_update_schib(sch)) { |
|
vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_NOT_OPER); |
|
rc = 0; |
|
goto out_unlock; |
|
} |
|
|
|
private = dev_get_drvdata(&sch->dev); |
|
if (private->state == VFIO_CCW_STATE_NOT_OPER) { |
|
private->state = private->mdev ? VFIO_CCW_STATE_IDLE : |
|
VFIO_CCW_STATE_STANDBY; |
|
} |
|
rc = 0; |
|
|
|
out_unlock: |
|
spin_unlock_irqrestore(sch->lock, flags); |
|
|
|
return rc; |
|
} |
|
|
|
static void vfio_ccw_queue_crw(struct vfio_ccw_private *private, |
|
unsigned int rsc, |
|
unsigned int erc, |
|
unsigned int rsid) |
|
{ |
|
struct vfio_ccw_crw *crw; |
|
|
|
/* |
|
* If unable to allocate a CRW, just drop the event and |
|
* carry on. The guest will either see a later one or |
|
* learn when it issues its own store subchannel. |
|
*/ |
|
crw = kzalloc(sizeof(*crw), GFP_ATOMIC); |
|
if (!crw) |
|
return; |
|
|
|
/* |
|
* Build the CRW based on the inputs given to us. |
|
*/ |
|
crw->crw.rsc = rsc; |
|
crw->crw.erc = erc; |
|
crw->crw.rsid = rsid; |
|
|
|
list_add_tail(&crw->next, &private->crw); |
|
queue_work(vfio_ccw_work_q, &private->crw_work); |
|
} |
|
|
|
static int vfio_ccw_chp_event(struct subchannel *sch, |
|
struct chp_link *link, int event) |
|
{ |
|
struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); |
|
int mask = chp_ssd_get_mask(&sch->ssd_info, link); |
|
int retry = 255; |
|
|
|
if (!private || !mask) |
|
return 0; |
|
|
|
trace_vfio_ccw_chp_event(private->sch->schid, mask, event); |
|
VFIO_CCW_MSG_EVENT(2, "%pUl (%x.%x.%04x): mask=0x%x event=%d\n", |
|
mdev_uuid(private->mdev), sch->schid.cssid, |
|
sch->schid.ssid, sch->schid.sch_no, |
|
mask, event); |
|
|
|
if (cio_update_schib(sch)) |
|
return -ENODEV; |
|
|
|
switch (event) { |
|
case CHP_VARY_OFF: |
|
/* Path logically turned off */ |
|
sch->opm &= ~mask; |
|
sch->lpm &= ~mask; |
|
if (sch->schib.pmcw.lpum & mask) |
|
cio_cancel_halt_clear(sch, &retry); |
|
break; |
|
case CHP_OFFLINE: |
|
/* Path is gone */ |
|
if (sch->schib.pmcw.lpum & mask) |
|
cio_cancel_halt_clear(sch, &retry); |
|
vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_PERRN, |
|
link->chpid.id); |
|
break; |
|
case CHP_VARY_ON: |
|
/* Path logically turned on */ |
|
sch->opm |= mask; |
|
sch->lpm |= mask; |
|
break; |
|
case CHP_ONLINE: |
|
/* Path became available */ |
|
sch->lpm |= mask & sch->opm; |
|
vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_INIT, |
|
link->chpid.id); |
|
break; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
static struct css_device_id vfio_ccw_sch_ids[] = { |
|
{ .match_flags = 0x1, .type = SUBCHANNEL_TYPE_IO, }, |
|
{ /* end of list */ }, |
|
}; |
|
MODULE_DEVICE_TABLE(css, vfio_ccw_sch_ids); |
|
|
|
static struct css_driver vfio_ccw_sch_driver = { |
|
.drv = { |
|
.name = "vfio_ccw", |
|
.owner = THIS_MODULE, |
|
}, |
|
.subchannel_type = vfio_ccw_sch_ids, |
|
.irq = vfio_ccw_sch_irq, |
|
.probe = vfio_ccw_sch_probe, |
|
.remove = vfio_ccw_sch_remove, |
|
.shutdown = vfio_ccw_sch_shutdown, |
|
.sch_event = vfio_ccw_sch_event, |
|
.chp_event = vfio_ccw_chp_event, |
|
}; |
|
|
|
static int __init vfio_ccw_debug_init(void) |
|
{ |
|
vfio_ccw_debug_msg_id = debug_register("vfio_ccw_msg", 16, 1, |
|
11 * sizeof(long)); |
|
if (!vfio_ccw_debug_msg_id) |
|
goto out_unregister; |
|
debug_register_view(vfio_ccw_debug_msg_id, &debug_sprintf_view); |
|
debug_set_level(vfio_ccw_debug_msg_id, 2); |
|
vfio_ccw_debug_trace_id = debug_register("vfio_ccw_trace", 16, 1, 16); |
|
if (!vfio_ccw_debug_trace_id) |
|
goto out_unregister; |
|
debug_register_view(vfio_ccw_debug_trace_id, &debug_hex_ascii_view); |
|
debug_set_level(vfio_ccw_debug_trace_id, 2); |
|
return 0; |
|
|
|
out_unregister: |
|
debug_unregister(vfio_ccw_debug_msg_id); |
|
debug_unregister(vfio_ccw_debug_trace_id); |
|
return -1; |
|
} |
|
|
|
static void vfio_ccw_debug_exit(void) |
|
{ |
|
debug_unregister(vfio_ccw_debug_msg_id); |
|
debug_unregister(vfio_ccw_debug_trace_id); |
|
} |
|
|
|
static void vfio_ccw_destroy_regions(void) |
|
{ |
|
kmem_cache_destroy(vfio_ccw_crw_region); |
|
kmem_cache_destroy(vfio_ccw_schib_region); |
|
kmem_cache_destroy(vfio_ccw_cmd_region); |
|
kmem_cache_destroy(vfio_ccw_io_region); |
|
} |
|
|
|
static int __init vfio_ccw_sch_init(void) |
|
{ |
|
int ret; |
|
|
|
ret = vfio_ccw_debug_init(); |
|
if (ret) |
|
return ret; |
|
|
|
vfio_ccw_work_q = create_singlethread_workqueue("vfio-ccw"); |
|
if (!vfio_ccw_work_q) { |
|
ret = -ENOMEM; |
|
goto out_err; |
|
} |
|
|
|
vfio_ccw_io_region = kmem_cache_create_usercopy("vfio_ccw_io_region", |
|
sizeof(struct ccw_io_region), 0, |
|
SLAB_ACCOUNT, 0, |
|
sizeof(struct ccw_io_region), NULL); |
|
if (!vfio_ccw_io_region) { |
|
ret = -ENOMEM; |
|
goto out_err; |
|
} |
|
|
|
vfio_ccw_cmd_region = kmem_cache_create_usercopy("vfio_ccw_cmd_region", |
|
sizeof(struct ccw_cmd_region), 0, |
|
SLAB_ACCOUNT, 0, |
|
sizeof(struct ccw_cmd_region), NULL); |
|
if (!vfio_ccw_cmd_region) { |
|
ret = -ENOMEM; |
|
goto out_err; |
|
} |
|
|
|
vfio_ccw_schib_region = kmem_cache_create_usercopy("vfio_ccw_schib_region", |
|
sizeof(struct ccw_schib_region), 0, |
|
SLAB_ACCOUNT, 0, |
|
sizeof(struct ccw_schib_region), NULL); |
|
|
|
if (!vfio_ccw_schib_region) { |
|
ret = -ENOMEM; |
|
goto out_err; |
|
} |
|
|
|
vfio_ccw_crw_region = kmem_cache_create_usercopy("vfio_ccw_crw_region", |
|
sizeof(struct ccw_crw_region), 0, |
|
SLAB_ACCOUNT, 0, |
|
sizeof(struct ccw_crw_region), NULL); |
|
|
|
if (!vfio_ccw_crw_region) { |
|
ret = -ENOMEM; |
|
goto out_err; |
|
} |
|
|
|
isc_register(VFIO_CCW_ISC); |
|
ret = css_driver_register(&vfio_ccw_sch_driver); |
|
if (ret) { |
|
isc_unregister(VFIO_CCW_ISC); |
|
goto out_err; |
|
} |
|
|
|
return ret; |
|
|
|
out_err: |
|
vfio_ccw_destroy_regions(); |
|
destroy_workqueue(vfio_ccw_work_q); |
|
vfio_ccw_debug_exit(); |
|
return ret; |
|
} |
|
|
|
static void __exit vfio_ccw_sch_exit(void) |
|
{ |
|
css_driver_unregister(&vfio_ccw_sch_driver); |
|
isc_unregister(VFIO_CCW_ISC); |
|
vfio_ccw_destroy_regions(); |
|
destroy_workqueue(vfio_ccw_work_q); |
|
vfio_ccw_debug_exit(); |
|
} |
|
module_init(vfio_ccw_sch_init); |
|
module_exit(vfio_ccw_sch_exit); |
|
|
|
MODULE_LICENSE("GPL v2");
|
|
|