mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
493 lines
12 KiB
493 lines
12 KiB
// SPDX-License-Identifier: GPL-2.0-or-later |
|
/* |
|
* VAS user space API for its accelerators (Only NX-GZIP is supported now) |
|
* Copyright (C) 2019 Haren Myneni, IBM Corp |
|
*/ |
|
|
|
#include <linux/kernel.h> |
|
#include <linux/device.h> |
|
#include <linux/cdev.h> |
|
#include <linux/fs.h> |
|
#include <linux/slab.h> |
|
#include <linux/uaccess.h> |
|
#include <linux/kthread.h> |
|
#include <linux/sched/signal.h> |
|
#include <linux/mmu_context.h> |
|
#include <linux/io.h> |
|
#include <asm/vas.h> |
|
#include <uapi/asm/vas-api.h> |
|
|
|
/* |
|
* The driver creates the device node that can be used as follows: |
|
* For NX-GZIP |
|
* |
|
* fd = open("/dev/crypto/nx-gzip", O_RDWR); |
|
* rc = ioctl(fd, VAS_TX_WIN_OPEN, &attr); |
|
* paste_addr = mmap(NULL, PAGE_SIZE, prot, MAP_SHARED, fd, 0ULL). |
|
* vas_copy(&crb, 0, 1); |
|
* vas_paste(paste_addr, 0, 1); |
|
* close(fd) or exit process to close window. |
|
* |
|
* where "vas_copy" and "vas_paste" are defined in copy-paste.h. |
|
* copy/paste returns to the user space directly. So refer NX hardware |
|
* documententation for exact copy/paste usage and completion / error |
|
* conditions. |
|
*/ |
|
|
|
/* |
|
* Wrapper object for the nx-gzip device - there is just one instance of |
|
* this node for the whole system. |
|
*/ |
|
static struct coproc_dev { |
|
struct cdev cdev; |
|
struct device *device; |
|
char *name; |
|
dev_t devt; |
|
struct class *class; |
|
enum vas_cop_type cop_type; |
|
const struct vas_user_win_ops *vops; |
|
} coproc_device; |
|
|
|
struct coproc_instance { |
|
struct coproc_dev *coproc; |
|
struct vas_window *txwin; |
|
}; |
|
|
|
static char *coproc_devnode(struct device *dev, umode_t *mode) |
|
{ |
|
return kasprintf(GFP_KERNEL, "crypto/%s", dev_name(dev)); |
|
} |
|
|
|
/* |
|
* Take reference to pid and mm |
|
*/ |
|
int get_vas_user_win_ref(struct vas_user_win_ref *task_ref) |
|
{ |
|
/* |
|
* Window opened by a child thread may not be closed when |
|
* it exits. So take reference to its pid and release it |
|
* when the window is free by parent thread. |
|
* Acquire a reference to the task's pid to make sure |
|
* pid will not be re-used - needed only for multithread |
|
* applications. |
|
*/ |
|
task_ref->pid = get_task_pid(current, PIDTYPE_PID); |
|
/* |
|
* Acquire a reference to the task's mm. |
|
*/ |
|
task_ref->mm = get_task_mm(current); |
|
if (!task_ref->mm) { |
|
put_pid(task_ref->pid); |
|
pr_err("VAS: pid(%d): mm_struct is not found\n", |
|
current->pid); |
|
return -EPERM; |
|
} |
|
|
|
mmgrab(task_ref->mm); |
|
mmput(task_ref->mm); |
|
/* |
|
* Process closes window during exit. In the case of |
|
* multithread application, the child thread can open |
|
* window and can exit without closing it. So takes tgid |
|
* reference until window closed to make sure tgid is not |
|
* reused. |
|
*/ |
|
task_ref->tgid = find_get_pid(task_tgid_vnr(current)); |
|
|
|
return 0; |
|
} |
|
|
|
/* |
|
* Successful return must release the task reference with |
|
* put_task_struct |
|
*/ |
|
static bool ref_get_pid_and_task(struct vas_user_win_ref *task_ref, |
|
struct task_struct **tskp, struct pid **pidp) |
|
{ |
|
struct task_struct *tsk; |
|
struct pid *pid; |
|
|
|
pid = task_ref->pid; |
|
tsk = get_pid_task(pid, PIDTYPE_PID); |
|
if (!tsk) { |
|
pid = task_ref->tgid; |
|
tsk = get_pid_task(pid, PIDTYPE_PID); |
|
/* |
|
* Parent thread (tgid) will be closing window when it |
|
* exits. So should not get here. |
|
*/ |
|
if (WARN_ON_ONCE(!tsk)) |
|
return false; |
|
} |
|
|
|
/* Return if the task is exiting. */ |
|
if (tsk->flags & PF_EXITING) { |
|
put_task_struct(tsk); |
|
return false; |
|
} |
|
|
|
*tskp = tsk; |
|
*pidp = pid; |
|
|
|
return true; |
|
} |
|
|
|
/* |
|
* Update the CSB to indicate a translation error. |
|
* |
|
* User space will be polling on CSB after the request is issued. |
|
* If NX can handle the request without any issues, it updates CSB. |
|
* Whereas if NX encounters page fault, the kernel will handle the |
|
* fault and update CSB with translation error. |
|
* |
|
* If we are unable to update the CSB means copy_to_user failed due to |
|
* invalid csb_addr, send a signal to the process. |
|
*/ |
|
void vas_update_csb(struct coprocessor_request_block *crb, |
|
struct vas_user_win_ref *task_ref) |
|
{ |
|
struct coprocessor_status_block csb; |
|
struct kernel_siginfo info; |
|
struct task_struct *tsk; |
|
void __user *csb_addr; |
|
struct pid *pid; |
|
int rc; |
|
|
|
/* |
|
* NX user space windows can not be opened for task->mm=NULL |
|
* and faults will not be generated for kernel requests. |
|
*/ |
|
if (WARN_ON_ONCE(!task_ref->mm)) |
|
return; |
|
|
|
csb_addr = (void __user *)be64_to_cpu(crb->csb_addr); |
|
|
|
memset(&csb, 0, sizeof(csb)); |
|
csb.cc = CSB_CC_FAULT_ADDRESS; |
|
csb.ce = CSB_CE_TERMINATION; |
|
csb.cs = 0; |
|
csb.count = 0; |
|
|
|
/* |
|
* NX operates and returns in BE format as defined CRB struct. |
|
* So saves fault_storage_addr in BE as NX pastes in FIFO and |
|
* expects user space to convert to CPU format. |
|
*/ |
|
csb.address = crb->stamp.nx.fault_storage_addr; |
|
csb.flags = 0; |
|
|
|
/* |
|
* Process closes send window after all pending NX requests are |
|
* completed. In multi-thread applications, a child thread can |
|
* open a window and can exit without closing it. May be some |
|
* requests are pending or this window can be used by other |
|
* threads later. We should handle faults if NX encounters |
|
* pages faults on these requests. Update CSB with translation |
|
* error and fault address. If csb_addr passed by user space is |
|
* invalid, send SEGV signal to pid saved in window. If the |
|
* child thread is not running, send the signal to tgid. |
|
* Parent thread (tgid) will close this window upon its exit. |
|
* |
|
* pid and mm references are taken when window is opened by |
|
* process (pid). So tgid is used only when child thread opens |
|
* a window and exits without closing it. |
|
*/ |
|
|
|
if (!ref_get_pid_and_task(task_ref, &tsk, &pid)) |
|
return; |
|
|
|
kthread_use_mm(task_ref->mm); |
|
rc = copy_to_user(csb_addr, &csb, sizeof(csb)); |
|
/* |
|
* User space polls on csb.flags (first byte). So add barrier |
|
* then copy first byte with csb flags update. |
|
*/ |
|
if (!rc) { |
|
csb.flags = CSB_V; |
|
/* Make sure update to csb.flags is visible now */ |
|
smp_mb(); |
|
rc = copy_to_user(csb_addr, &csb, sizeof(u8)); |
|
} |
|
kthread_unuse_mm(task_ref->mm); |
|
put_task_struct(tsk); |
|
|
|
/* Success */ |
|
if (!rc) |
|
return; |
|
|
|
|
|
pr_debug("Invalid CSB address 0x%p signalling pid(%d)\n", |
|
csb_addr, pid_vnr(pid)); |
|
|
|
clear_siginfo(&info); |
|
info.si_signo = SIGSEGV; |
|
info.si_errno = EFAULT; |
|
info.si_code = SEGV_MAPERR; |
|
info.si_addr = csb_addr; |
|
/* |
|
* process will be polling on csb.flags after request is sent to |
|
* NX. So generally CSB update should not fail except when an |
|
* application passes invalid csb_addr. So an error message will |
|
* be displayed and leave it to user space whether to ignore or |
|
* handle this signal. |
|
*/ |
|
rcu_read_lock(); |
|
rc = kill_pid_info(SIGSEGV, &info, pid); |
|
rcu_read_unlock(); |
|
|
|
pr_devel("%s(): pid %d kill_proc_info() rc %d\n", __func__, |
|
pid_vnr(pid), rc); |
|
} |
|
|
|
void vas_dump_crb(struct coprocessor_request_block *crb) |
|
{ |
|
struct data_descriptor_entry *dde; |
|
struct nx_fault_stamp *nx; |
|
|
|
dde = &crb->source; |
|
pr_devel("SrcDDE: addr 0x%llx, len %d, count %d, idx %d, flags %d\n", |
|
be64_to_cpu(dde->address), be32_to_cpu(dde->length), |
|
dde->count, dde->index, dde->flags); |
|
|
|
dde = &crb->target; |
|
pr_devel("TgtDDE: addr 0x%llx, len %d, count %d, idx %d, flags %d\n", |
|
be64_to_cpu(dde->address), be32_to_cpu(dde->length), |
|
dde->count, dde->index, dde->flags); |
|
|
|
nx = &crb->stamp.nx; |
|
pr_devel("NX Stamp: PSWID 0x%x, FSA 0x%llx, flags 0x%x, FS 0x%x\n", |
|
be32_to_cpu(nx->pswid), |
|
be64_to_cpu(crb->stamp.nx.fault_storage_addr), |
|
nx->flags, nx->fault_status); |
|
} |
|
|
|
static int coproc_open(struct inode *inode, struct file *fp) |
|
{ |
|
struct coproc_instance *cp_inst; |
|
|
|
cp_inst = kzalloc(sizeof(*cp_inst), GFP_KERNEL); |
|
if (!cp_inst) |
|
return -ENOMEM; |
|
|
|
cp_inst->coproc = container_of(inode->i_cdev, struct coproc_dev, |
|
cdev); |
|
fp->private_data = cp_inst; |
|
|
|
return 0; |
|
} |
|
|
|
static int coproc_ioc_tx_win_open(struct file *fp, unsigned long arg) |
|
{ |
|
void __user *uptr = (void __user *)arg; |
|
struct vas_tx_win_open_attr uattr; |
|
struct coproc_instance *cp_inst; |
|
struct vas_window *txwin; |
|
int rc; |
|
|
|
cp_inst = fp->private_data; |
|
|
|
/* |
|
* One window for file descriptor |
|
*/ |
|
if (cp_inst->txwin) |
|
return -EEXIST; |
|
|
|
rc = copy_from_user(&uattr, uptr, sizeof(uattr)); |
|
if (rc) { |
|
pr_err("%s(): copy_from_user() returns %d\n", __func__, rc); |
|
return -EFAULT; |
|
} |
|
|
|
if (uattr.version != 1) { |
|
pr_err("Invalid window open API version\n"); |
|
return -EINVAL; |
|
} |
|
|
|
if (!cp_inst->coproc->vops && !cp_inst->coproc->vops->open_win) { |
|
pr_err("VAS API is not registered\n"); |
|
return -EACCES; |
|
} |
|
|
|
txwin = cp_inst->coproc->vops->open_win(uattr.vas_id, uattr.flags, |
|
cp_inst->coproc->cop_type); |
|
if (IS_ERR(txwin)) { |
|
pr_err("%s() VAS window open failed, %ld\n", __func__, |
|
PTR_ERR(txwin)); |
|
return PTR_ERR(txwin); |
|
} |
|
|
|
cp_inst->txwin = txwin; |
|
|
|
return 0; |
|
} |
|
|
|
static int coproc_release(struct inode *inode, struct file *fp) |
|
{ |
|
struct coproc_instance *cp_inst = fp->private_data; |
|
int rc; |
|
|
|
if (cp_inst->txwin) { |
|
if (cp_inst->coproc->vops && |
|
cp_inst->coproc->vops->close_win) { |
|
rc = cp_inst->coproc->vops->close_win(cp_inst->txwin); |
|
if (rc) |
|
return rc; |
|
} |
|
cp_inst->txwin = NULL; |
|
} |
|
|
|
kfree(cp_inst); |
|
fp->private_data = NULL; |
|
|
|
/* |
|
* We don't know here if user has other receive windows |
|
* open, so we can't really call clear_thread_tidr(). |
|
* So, once the process calls set_thread_tidr(), the |
|
* TIDR value sticks around until process exits, resulting |
|
* in an extra copy in restore_sprs(). |
|
*/ |
|
|
|
return 0; |
|
} |
|
|
|
static int coproc_mmap(struct file *fp, struct vm_area_struct *vma) |
|
{ |
|
struct coproc_instance *cp_inst = fp->private_data; |
|
struct vas_window *txwin; |
|
unsigned long pfn; |
|
u64 paste_addr; |
|
pgprot_t prot; |
|
int rc; |
|
|
|
txwin = cp_inst->txwin; |
|
|
|
if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { |
|
pr_debug("%s(): size 0x%zx, PAGE_SIZE 0x%zx\n", __func__, |
|
(vma->vm_end - vma->vm_start), PAGE_SIZE); |
|
return -EINVAL; |
|
} |
|
|
|
/* Ensure instance has an open send window */ |
|
if (!txwin) { |
|
pr_err("%s(): No send window open?\n", __func__); |
|
return -EINVAL; |
|
} |
|
|
|
if (!cp_inst->coproc->vops && !cp_inst->coproc->vops->paste_addr) { |
|
pr_err("%s(): VAS API is not registered\n", __func__); |
|
return -EACCES; |
|
} |
|
|
|
paste_addr = cp_inst->coproc->vops->paste_addr(txwin); |
|
if (!paste_addr) { |
|
pr_err("%s(): Window paste address failed\n", __func__); |
|
return -EINVAL; |
|
} |
|
|
|
pfn = paste_addr >> PAGE_SHIFT; |
|
|
|
/* flags, page_prot from cxl_mmap(), except we want cachable */ |
|
vma->vm_flags |= VM_IO | VM_PFNMAP; |
|
vma->vm_page_prot = pgprot_cached(vma->vm_page_prot); |
|
|
|
prot = __pgprot(pgprot_val(vma->vm_page_prot) | _PAGE_DIRTY); |
|
|
|
rc = remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, |
|
vma->vm_end - vma->vm_start, prot); |
|
|
|
pr_devel("%s(): paste addr %llx at %lx, rc %d\n", __func__, |
|
paste_addr, vma->vm_start, rc); |
|
|
|
return rc; |
|
} |
|
|
|
static long coproc_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) |
|
{ |
|
switch (cmd) { |
|
case VAS_TX_WIN_OPEN: |
|
return coproc_ioc_tx_win_open(fp, arg); |
|
default: |
|
return -EINVAL; |
|
} |
|
} |
|
|
|
static struct file_operations coproc_fops = { |
|
.open = coproc_open, |
|
.release = coproc_release, |
|
.mmap = coproc_mmap, |
|
.unlocked_ioctl = coproc_ioctl, |
|
}; |
|
|
|
/* |
|
* Supporting only nx-gzip coprocessor type now, but this API code |
|
* extended to other coprocessor types later. |
|
*/ |
|
int vas_register_coproc_api(struct module *mod, enum vas_cop_type cop_type, |
|
const char *name, |
|
const struct vas_user_win_ops *vops) |
|
{ |
|
int rc = -EINVAL; |
|
dev_t devno; |
|
|
|
rc = alloc_chrdev_region(&coproc_device.devt, 1, 1, name); |
|
if (rc) { |
|
pr_err("Unable to allocate coproc major number: %i\n", rc); |
|
return rc; |
|
} |
|
|
|
pr_devel("%s device allocated, dev [%i,%i]\n", name, |
|
MAJOR(coproc_device.devt), MINOR(coproc_device.devt)); |
|
|
|
coproc_device.class = class_create(mod, name); |
|
if (IS_ERR(coproc_device.class)) { |
|
rc = PTR_ERR(coproc_device.class); |
|
pr_err("Unable to create %s class %d\n", name, rc); |
|
goto err_class; |
|
} |
|
coproc_device.class->devnode = coproc_devnode; |
|
coproc_device.cop_type = cop_type; |
|
coproc_device.vops = vops; |
|
|
|
coproc_fops.owner = mod; |
|
cdev_init(&coproc_device.cdev, &coproc_fops); |
|
|
|
devno = MKDEV(MAJOR(coproc_device.devt), 0); |
|
rc = cdev_add(&coproc_device.cdev, devno, 1); |
|
if (rc) { |
|
pr_err("cdev_add() failed %d\n", rc); |
|
goto err_cdev; |
|
} |
|
|
|
coproc_device.device = device_create(coproc_device.class, NULL, |
|
devno, NULL, name, MINOR(devno)); |
|
if (IS_ERR(coproc_device.device)) { |
|
rc = PTR_ERR(coproc_device.device); |
|
pr_err("Unable to create coproc-%d %d\n", MINOR(devno), rc); |
|
goto err; |
|
} |
|
|
|
pr_devel("%s: Added dev [%d,%d]\n", __func__, MAJOR(devno), |
|
MINOR(devno)); |
|
|
|
return 0; |
|
|
|
err: |
|
cdev_del(&coproc_device.cdev); |
|
err_cdev: |
|
class_destroy(coproc_device.class); |
|
err_class: |
|
unregister_chrdev_region(coproc_device.devt, 1); |
|
return rc; |
|
} |
|
|
|
void vas_unregister_coproc_api(void) |
|
{ |
|
dev_t devno; |
|
|
|
cdev_del(&coproc_device.cdev); |
|
devno = MKDEV(MAJOR(coproc_device.devt), 0); |
|
device_destroy(coproc_device.class, devno); |
|
|
|
class_destroy(coproc_device.class); |
|
unregister_chrdev_region(coproc_device.devt, 1); |
|
}
|
|
|