mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
805 lines
23 KiB
805 lines
23 KiB
/* |
|
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. |
|
* |
|
* This software is available to you under a choice of one of two |
|
* licenses. You may choose to be licensed under the terms of the GNU |
|
* General Public License (GPL) Version 2, available from the file |
|
* COPYING in the main directory of this source tree, or the |
|
* OpenIB.org BSD license below: |
|
* |
|
* Redistribution and use in source and binary forms, with or |
|
* without modification, are permitted provided that the following |
|
* conditions are met: |
|
* |
|
* - Redistributions of source code must retain the above |
|
* copyright notice, this list of conditions and the following |
|
* disclaimer. |
|
* |
|
* - Redistributions in binary form must reproduce the above |
|
* copyright notice, this list of conditions and the following |
|
* disclaimer in the documentation and/or other materials |
|
* provided with the distribution. |
|
* |
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
|
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
|
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
|
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
|
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
|
* SOFTWARE. |
|
*/ |
|
|
|
#include <rdma/rdma_user_ioctl.h> |
|
#include <rdma/uverbs_ioctl.h> |
|
#include "rdma_core.h" |
|
#include "uverbs.h" |
|
|
|
struct bundle_alloc_head { |
|
struct bundle_alloc_head *next; |
|
u8 data[]; |
|
}; |
|
|
|
struct bundle_priv { |
|
/* Must be first */ |
|
struct bundle_alloc_head alloc_head; |
|
struct bundle_alloc_head *allocated_mem; |
|
size_t internal_avail; |
|
size_t internal_used; |
|
|
|
struct radix_tree_root *radix; |
|
const struct uverbs_api_ioctl_method *method_elm; |
|
void __rcu **radix_slots; |
|
unsigned long radix_slots_len; |
|
u32 method_key; |
|
|
|
struct ib_uverbs_attr __user *user_attrs; |
|
struct ib_uverbs_attr *uattrs; |
|
|
|
DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN); |
|
DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN); |
|
DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN); |
|
|
|
/* |
|
* Must be last. bundle ends in a flex array which overlaps |
|
* internal_buffer. |
|
*/ |
|
struct uverbs_attr_bundle bundle; |
|
u64 internal_buffer[32]; |
|
}; |
|
|
|
/* |
|
* Each method has an absolute minimum amount of memory it needs to allocate, |
|
* precompute that amount and determine if the onstack memory can be used or |
|
* if allocation is need. |
|
*/ |
|
void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm, |
|
unsigned int num_attrs) |
|
{ |
|
struct bundle_priv *pbundle; |
|
size_t bundle_size = |
|
offsetof(struct bundle_priv, internal_buffer) + |
|
sizeof(*pbundle->bundle.attrs) * method_elm->key_bitmap_len + |
|
sizeof(*pbundle->uattrs) * num_attrs; |
|
|
|
method_elm->use_stack = bundle_size <= sizeof(*pbundle); |
|
method_elm->bundle_size = |
|
ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer)); |
|
|
|
/* Do not want order-2 allocations for this. */ |
|
WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE); |
|
} |
|
|
|
/** |
|
* _uverbs_alloc() - Quickly allocate memory for use with a bundle |
|
* @bundle: The bundle |
|
* @size: Number of bytes to allocate |
|
* @flags: Allocator flags |
|
* |
|
* The bundle allocator is intended for allocations that are connected with |
|
* processing the system call related to the bundle. The allocated memory is |
|
* always freed once the system call completes, and cannot be freed any other |
|
* way. |
|
* |
|
* This tries to use a small pool of pre-allocated memory for performance. |
|
*/ |
|
__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size, |
|
gfp_t flags) |
|
{ |
|
struct bundle_priv *pbundle = |
|
container_of(bundle, struct bundle_priv, bundle); |
|
size_t new_used; |
|
void *res; |
|
|
|
if (check_add_overflow(size, pbundle->internal_used, &new_used)) |
|
return ERR_PTR(-EOVERFLOW); |
|
|
|
if (new_used > pbundle->internal_avail) { |
|
struct bundle_alloc_head *buf; |
|
|
|
buf = kvmalloc(struct_size(buf, data, size), flags); |
|
if (!buf) |
|
return ERR_PTR(-ENOMEM); |
|
buf->next = pbundle->allocated_mem; |
|
pbundle->allocated_mem = buf; |
|
return buf->data; |
|
} |
|
|
|
res = (void *)pbundle->internal_buffer + pbundle->internal_used; |
|
pbundle->internal_used = |
|
ALIGN(new_used, sizeof(*pbundle->internal_buffer)); |
|
if (want_init_on_alloc(flags)) |
|
memset(res, 0, size); |
|
return res; |
|
} |
|
EXPORT_SYMBOL(_uverbs_alloc); |
|
|
|
static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr, |
|
u16 len) |
|
{ |
|
if (uattr->len > sizeof_field(struct ib_uverbs_attr, data)) |
|
return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len, |
|
uattr->len - len); |
|
|
|
return !memchr_inv((const void *)&uattr->data + len, |
|
0, uattr->len - len); |
|
} |
|
|
|
static int uverbs_set_output(const struct uverbs_attr_bundle *bundle, |
|
const struct uverbs_attr *attr) |
|
{ |
|
struct bundle_priv *pbundle = |
|
container_of(bundle, struct bundle_priv, bundle); |
|
u16 flags; |
|
|
|
flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags | |
|
UVERBS_ATTR_F_VALID_OUTPUT; |
|
if (put_user(flags, |
|
&pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags)) |
|
return -EFAULT; |
|
return 0; |
|
} |
|
|
|
static int uverbs_process_idrs_array(struct bundle_priv *pbundle, |
|
const struct uverbs_api_attr *attr_uapi, |
|
struct uverbs_objs_arr_attr *attr, |
|
struct ib_uverbs_attr *uattr, |
|
u32 attr_bkey) |
|
{ |
|
const struct uverbs_attr_spec *spec = &attr_uapi->spec; |
|
size_t array_len; |
|
u32 *idr_vals; |
|
int ret = 0; |
|
size_t i; |
|
|
|
if (uattr->attr_data.reserved) |
|
return -EINVAL; |
|
|
|
if (uattr->len % sizeof(u32)) |
|
return -EINVAL; |
|
|
|
array_len = uattr->len / sizeof(u32); |
|
if (array_len < spec->u2.objs_arr.min_len || |
|
array_len > spec->u2.objs_arr.max_len) |
|
return -EINVAL; |
|
|
|
attr->uobjects = |
|
uverbs_alloc(&pbundle->bundle, |
|
array_size(array_len, sizeof(*attr->uobjects))); |
|
if (IS_ERR(attr->uobjects)) |
|
return PTR_ERR(attr->uobjects); |
|
|
|
/* |
|
* Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects |
|
* to store idrs array and avoid additional memory allocation. The |
|
* idrs array is offset to the end of the uobjects array so we will be |
|
* able to read idr and replace with a pointer. |
|
*/ |
|
idr_vals = (u32 *)(attr->uobjects + array_len) - array_len; |
|
|
|
if (uattr->len > sizeof(uattr->data)) { |
|
ret = copy_from_user(idr_vals, u64_to_user_ptr(uattr->data), |
|
uattr->len); |
|
if (ret) |
|
return -EFAULT; |
|
} else { |
|
memcpy(idr_vals, &uattr->data, uattr->len); |
|
} |
|
|
|
for (i = 0; i != array_len; i++) { |
|
attr->uobjects[i] = uverbs_get_uobject_from_file( |
|
spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access, |
|
idr_vals[i], &pbundle->bundle); |
|
if (IS_ERR(attr->uobjects[i])) { |
|
ret = PTR_ERR(attr->uobjects[i]); |
|
break; |
|
} |
|
} |
|
|
|
attr->len = i; |
|
__set_bit(attr_bkey, pbundle->spec_finalize); |
|
return ret; |
|
} |
|
|
|
static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi, |
|
struct uverbs_objs_arr_attr *attr, |
|
bool commit, |
|
struct uverbs_attr_bundle *attrs) |
|
{ |
|
const struct uverbs_attr_spec *spec = &attr_uapi->spec; |
|
size_t i; |
|
|
|
for (i = 0; i != attr->len; i++) |
|
uverbs_finalize_object(attr->uobjects[i], |
|
spec->u2.objs_arr.access, false, commit, |
|
attrs); |
|
} |
|
|
|
static int uverbs_process_attr(struct bundle_priv *pbundle, |
|
const struct uverbs_api_attr *attr_uapi, |
|
struct ib_uverbs_attr *uattr, u32 attr_bkey) |
|
{ |
|
const struct uverbs_attr_spec *spec = &attr_uapi->spec; |
|
struct uverbs_attr *e = &pbundle->bundle.attrs[attr_bkey]; |
|
const struct uverbs_attr_spec *val_spec = spec; |
|
struct uverbs_obj_attr *o_attr; |
|
|
|
switch (spec->type) { |
|
case UVERBS_ATTR_TYPE_ENUM_IN: |
|
if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems) |
|
return -EOPNOTSUPP; |
|
|
|
if (uattr->attr_data.enum_data.reserved) |
|
return -EINVAL; |
|
|
|
val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id]; |
|
|
|
/* Currently we only support PTR_IN based enums */ |
|
if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN) |
|
return -EOPNOTSUPP; |
|
|
|
e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id; |
|
fallthrough; |
|
case UVERBS_ATTR_TYPE_PTR_IN: |
|
/* Ensure that any data provided by userspace beyond the known |
|
* struct is zero. Userspace that knows how to use some future |
|
* longer struct will fail here if used with an old kernel and |
|
* non-zero content, making ABI compat/discovery simpler. |
|
*/ |
|
if (uattr->len > val_spec->u.ptr.len && |
|
val_spec->zero_trailing && |
|
!uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len)) |
|
return -EOPNOTSUPP; |
|
|
|
fallthrough; |
|
case UVERBS_ATTR_TYPE_PTR_OUT: |
|
if (uattr->len < val_spec->u.ptr.min_len || |
|
(!val_spec->zero_trailing && |
|
uattr->len > val_spec->u.ptr.len)) |
|
return -EINVAL; |
|
|
|
if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN && |
|
uattr->attr_data.reserved) |
|
return -EINVAL; |
|
|
|
e->ptr_attr.uattr_idx = uattr - pbundle->uattrs; |
|
e->ptr_attr.len = uattr->len; |
|
|
|
if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) { |
|
void *p; |
|
|
|
p = uverbs_alloc(&pbundle->bundle, uattr->len); |
|
if (IS_ERR(p)) |
|
return PTR_ERR(p); |
|
|
|
e->ptr_attr.ptr = p; |
|
|
|
if (copy_from_user(p, u64_to_user_ptr(uattr->data), |
|
uattr->len)) |
|
return -EFAULT; |
|
} else { |
|
e->ptr_attr.data = uattr->data; |
|
} |
|
break; |
|
|
|
case UVERBS_ATTR_TYPE_IDR: |
|
case UVERBS_ATTR_TYPE_FD: |
|
if (uattr->attr_data.reserved) |
|
return -EINVAL; |
|
|
|
if (uattr->len != 0) |
|
return -EINVAL; |
|
|
|
o_attr = &e->obj_attr; |
|
o_attr->attr_elm = attr_uapi; |
|
|
|
/* |
|
* The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and |
|
* s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64 |
|
* here without caring about truncation as we know that the |
|
* IDR implementation today rejects negative IDs |
|
*/ |
|
o_attr->uobject = uverbs_get_uobject_from_file( |
|
spec->u.obj.obj_type, spec->u.obj.access, |
|
uattr->data_s64, &pbundle->bundle); |
|
if (IS_ERR(o_attr->uobject)) |
|
return PTR_ERR(o_attr->uobject); |
|
__set_bit(attr_bkey, pbundle->uobj_finalize); |
|
|
|
if (spec->u.obj.access == UVERBS_ACCESS_NEW) { |
|
unsigned int uattr_idx = uattr - pbundle->uattrs; |
|
s64 id = o_attr->uobject->id; |
|
|
|
/* Copy the allocated id to the user-space */ |
|
if (put_user(id, &pbundle->user_attrs[uattr_idx].data)) |
|
return -EFAULT; |
|
} |
|
|
|
break; |
|
|
|
case UVERBS_ATTR_TYPE_IDRS_ARRAY: |
|
return uverbs_process_idrs_array(pbundle, attr_uapi, |
|
&e->objs_arr_attr, uattr, |
|
attr_bkey); |
|
default: |
|
return -EOPNOTSUPP; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
/* |
|
* We search the radix tree with the method prefix and now we want to fast |
|
* search the suffix bits to get a particular attribute pointer. It is not |
|
* totally clear to me if this breaks the radix tree encasulation or not, but |
|
* it uses the iter data to determine if the method iter points at the same |
|
* chunk that will store the attribute, if so it just derefs it directly. By |
|
* construction in most kernel configs the method and attrs will all fit in a |
|
* single radix chunk, so in most cases this will have no search. Other cases |
|
* this falls back to a full search. |
|
*/ |
|
static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle, |
|
u32 attr_key) |
|
{ |
|
void __rcu **slot; |
|
|
|
if (likely(attr_key < pbundle->radix_slots_len)) { |
|
void *entry; |
|
|
|
slot = pbundle->radix_slots + attr_key; |
|
entry = rcu_dereference_raw(*slot); |
|
if (likely(!radix_tree_is_internal_node(entry) && entry)) |
|
return slot; |
|
} |
|
|
|
return radix_tree_lookup_slot(pbundle->radix, |
|
pbundle->method_key | attr_key); |
|
} |
|
|
|
static int uverbs_set_attr(struct bundle_priv *pbundle, |
|
struct ib_uverbs_attr *uattr) |
|
{ |
|
u32 attr_key = uapi_key_attr(uattr->attr_id); |
|
u32 attr_bkey = uapi_bkey_attr(attr_key); |
|
const struct uverbs_api_attr *attr; |
|
void __rcu **slot; |
|
int ret; |
|
|
|
slot = uapi_get_attr_for_method(pbundle, attr_key); |
|
if (!slot) { |
|
/* |
|
* Kernel does not support the attribute but user-space says it |
|
* is mandatory |
|
*/ |
|
if (uattr->flags & UVERBS_ATTR_F_MANDATORY) |
|
return -EPROTONOSUPPORT; |
|
return 0; |
|
} |
|
attr = rcu_dereference_protected(*slot, true); |
|
|
|
/* Reject duplicate attributes from user-space */ |
|
if (test_bit(attr_bkey, pbundle->bundle.attr_present)) |
|
return -EINVAL; |
|
|
|
ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey); |
|
if (ret) |
|
return ret; |
|
|
|
__set_bit(attr_bkey, pbundle->bundle.attr_present); |
|
|
|
return 0; |
|
} |
|
|
|
static int ib_uverbs_run_method(struct bundle_priv *pbundle, |
|
unsigned int num_attrs) |
|
{ |
|
int (*handler)(struct uverbs_attr_bundle *attrs); |
|
size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs); |
|
unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey; |
|
unsigned int i; |
|
int ret; |
|
|
|
/* See uverbs_disassociate_api() */ |
|
handler = srcu_dereference( |
|
pbundle->method_elm->handler, |
|
&pbundle->bundle.ufile->device->disassociate_srcu); |
|
if (!handler) |
|
return -EIO; |
|
|
|
pbundle->uattrs = uverbs_alloc(&pbundle->bundle, uattrs_size); |
|
if (IS_ERR(pbundle->uattrs)) |
|
return PTR_ERR(pbundle->uattrs); |
|
if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size)) |
|
return -EFAULT; |
|
|
|
for (i = 0; i != num_attrs; i++) { |
|
ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]); |
|
if (unlikely(ret)) |
|
return ret; |
|
} |
|
|
|
/* User space did not provide all the mandatory attributes */ |
|
if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory, |
|
pbundle->bundle.attr_present, |
|
pbundle->method_elm->key_bitmap_len))) |
|
return -EINVAL; |
|
|
|
if (pbundle->method_elm->has_udata) |
|
uverbs_fill_udata(&pbundle->bundle, |
|
&pbundle->bundle.driver_udata, |
|
UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT); |
|
else |
|
pbundle->bundle.driver_udata = (struct ib_udata){}; |
|
|
|
if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) { |
|
struct uverbs_obj_attr *destroy_attr = |
|
&pbundle->bundle.attrs[destroy_bkey].obj_attr; |
|
|
|
ret = uobj_destroy(destroy_attr->uobject, &pbundle->bundle); |
|
if (ret) |
|
return ret; |
|
__clear_bit(destroy_bkey, pbundle->uobj_finalize); |
|
|
|
ret = handler(&pbundle->bundle); |
|
uobj_put_destroy(destroy_attr->uobject); |
|
} else { |
|
ret = handler(&pbundle->bundle); |
|
} |
|
|
|
/* |
|
* Until the drivers are revised to use the bundle directly we have to |
|
* assume that the driver wrote to its UHW_OUT and flag userspace |
|
* appropriately. |
|
*/ |
|
if (!ret && pbundle->method_elm->has_udata) { |
|
const struct uverbs_attr *attr = |
|
uverbs_attr_get(&pbundle->bundle, UVERBS_ATTR_UHW_OUT); |
|
|
|
if (!IS_ERR(attr)) |
|
ret = uverbs_set_output(&pbundle->bundle, attr); |
|
} |
|
|
|
/* |
|
* EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can |
|
* not invoke the method because the request is not supported. No |
|
* other cases should return this code. |
|
*/ |
|
if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT)) |
|
return -EINVAL; |
|
|
|
return ret; |
|
} |
|
|
|
static void bundle_destroy(struct bundle_priv *pbundle, bool commit) |
|
{ |
|
unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len; |
|
struct bundle_alloc_head *memblock; |
|
unsigned int i; |
|
|
|
/* fast path for simple uobjects */ |
|
i = -1; |
|
while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len, |
|
i + 1)) < key_bitmap_len) { |
|
struct uverbs_attr *attr = &pbundle->bundle.attrs[i]; |
|
|
|
uverbs_finalize_object( |
|
attr->obj_attr.uobject, |
|
attr->obj_attr.attr_elm->spec.u.obj.access, |
|
test_bit(i, pbundle->uobj_hw_obj_valid), |
|
commit, |
|
&pbundle->bundle); |
|
} |
|
|
|
i = -1; |
|
while ((i = find_next_bit(pbundle->spec_finalize, key_bitmap_len, |
|
i + 1)) < key_bitmap_len) { |
|
struct uverbs_attr *attr = &pbundle->bundle.attrs[i]; |
|
const struct uverbs_api_attr *attr_uapi; |
|
void __rcu **slot; |
|
|
|
slot = uapi_get_attr_for_method( |
|
pbundle, |
|
pbundle->method_key | uapi_bkey_to_key_attr(i)); |
|
if (WARN_ON(!slot)) |
|
continue; |
|
|
|
attr_uapi = rcu_dereference_protected(*slot, true); |
|
|
|
if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) { |
|
uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr, |
|
commit, &pbundle->bundle); |
|
} |
|
} |
|
|
|
for (memblock = pbundle->allocated_mem; memblock;) { |
|
struct bundle_alloc_head *tmp = memblock; |
|
|
|
memblock = memblock->next; |
|
kvfree(tmp); |
|
} |
|
} |
|
|
|
static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile, |
|
struct ib_uverbs_ioctl_hdr *hdr, |
|
struct ib_uverbs_attr __user *user_attrs) |
|
{ |
|
const struct uverbs_api_ioctl_method *method_elm; |
|
struct uverbs_api *uapi = ufile->device->uapi; |
|
struct radix_tree_iter attrs_iter; |
|
struct bundle_priv *pbundle; |
|
struct bundle_priv onstack; |
|
void __rcu **slot; |
|
int ret; |
|
|
|
if (unlikely(hdr->driver_id != uapi->driver_id)) |
|
return -EINVAL; |
|
|
|
slot = radix_tree_iter_lookup( |
|
&uapi->radix, &attrs_iter, |
|
uapi_key_obj(hdr->object_id) | |
|
uapi_key_ioctl_method(hdr->method_id)); |
|
if (unlikely(!slot)) |
|
return -EPROTONOSUPPORT; |
|
method_elm = rcu_dereference_protected(*slot, true); |
|
|
|
if (!method_elm->use_stack) { |
|
pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL); |
|
if (!pbundle) |
|
return -ENOMEM; |
|
pbundle->internal_avail = |
|
method_elm->bundle_size - |
|
offsetof(struct bundle_priv, internal_buffer); |
|
pbundle->alloc_head.next = NULL; |
|
pbundle->allocated_mem = &pbundle->alloc_head; |
|
} else { |
|
pbundle = &onstack; |
|
pbundle->internal_avail = sizeof(pbundle->internal_buffer); |
|
pbundle->allocated_mem = NULL; |
|
} |
|
|
|
/* Space for the pbundle->bundle.attrs flex array */ |
|
pbundle->method_elm = method_elm; |
|
pbundle->method_key = attrs_iter.index; |
|
pbundle->bundle.ufile = ufile; |
|
pbundle->bundle.context = NULL; /* only valid if bundle has uobject */ |
|
pbundle->radix = &uapi->radix; |
|
pbundle->radix_slots = slot; |
|
pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter); |
|
pbundle->user_attrs = user_attrs; |
|
|
|
pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len * |
|
sizeof(*pbundle->bundle.attrs), |
|
sizeof(*pbundle->internal_buffer)); |
|
memset(pbundle->bundle.attr_present, 0, |
|
sizeof(pbundle->bundle.attr_present)); |
|
memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize)); |
|
memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize)); |
|
memset(pbundle->uobj_hw_obj_valid, 0, |
|
sizeof(pbundle->uobj_hw_obj_valid)); |
|
|
|
ret = ib_uverbs_run_method(pbundle, hdr->num_attrs); |
|
bundle_destroy(pbundle, ret == 0); |
|
return ret; |
|
} |
|
|
|
long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) |
|
{ |
|
struct ib_uverbs_file *file = filp->private_data; |
|
struct ib_uverbs_ioctl_hdr __user *user_hdr = |
|
(struct ib_uverbs_ioctl_hdr __user *)arg; |
|
struct ib_uverbs_ioctl_hdr hdr; |
|
int srcu_key; |
|
int err; |
|
|
|
if (unlikely(cmd != RDMA_VERBS_IOCTL)) |
|
return -ENOIOCTLCMD; |
|
|
|
err = copy_from_user(&hdr, user_hdr, sizeof(hdr)); |
|
if (err) |
|
return -EFAULT; |
|
|
|
if (hdr.length > PAGE_SIZE || |
|
hdr.length != struct_size(&hdr, attrs, hdr.num_attrs)) |
|
return -EINVAL; |
|
|
|
if (hdr.reserved1 || hdr.reserved2) |
|
return -EPROTONOSUPPORT; |
|
|
|
srcu_key = srcu_read_lock(&file->device->disassociate_srcu); |
|
err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs); |
|
srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); |
|
return err; |
|
} |
|
|
|
int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle, |
|
size_t idx, u64 allowed_bits) |
|
{ |
|
const struct uverbs_attr *attr; |
|
u64 flags; |
|
|
|
attr = uverbs_attr_get(attrs_bundle, idx); |
|
/* Missing attribute means 0 flags */ |
|
if (IS_ERR(attr)) { |
|
*to = 0; |
|
return 0; |
|
} |
|
|
|
/* |
|
* New userspace code should use 8 bytes to pass flags, but we |
|
* transparently support old userspaces that were using 4 bytes as |
|
* well. |
|
*/ |
|
if (attr->ptr_attr.len == 8) |
|
flags = attr->ptr_attr.data; |
|
else if (attr->ptr_attr.len == 4) |
|
flags = *(u32 *)&attr->ptr_attr.data; |
|
else |
|
return -EINVAL; |
|
|
|
if (flags & ~allowed_bits) |
|
return -EINVAL; |
|
|
|
*to = flags; |
|
return 0; |
|
} |
|
EXPORT_SYMBOL(uverbs_get_flags64); |
|
|
|
int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, |
|
size_t idx, u64 allowed_bits) |
|
{ |
|
u64 flags; |
|
int ret; |
|
|
|
ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits); |
|
if (ret) |
|
return ret; |
|
|
|
if (flags > U32_MAX) |
|
return -EINVAL; |
|
*to = flags; |
|
|
|
return 0; |
|
} |
|
EXPORT_SYMBOL(uverbs_get_flags32); |
|
|
|
/* |
|
* Fill a ib_udata struct (core or uhw) using the given attribute IDs. |
|
* This is primarily used to convert the UVERBS_ATTR_UHW() into the |
|
* ib_udata format used by the drivers. |
|
*/ |
|
void uverbs_fill_udata(struct uverbs_attr_bundle *bundle, |
|
struct ib_udata *udata, unsigned int attr_in, |
|
unsigned int attr_out) |
|
{ |
|
struct bundle_priv *pbundle = |
|
container_of(bundle, struct bundle_priv, bundle); |
|
const struct uverbs_attr *in = |
|
uverbs_attr_get(&pbundle->bundle, attr_in); |
|
const struct uverbs_attr *out = |
|
uverbs_attr_get(&pbundle->bundle, attr_out); |
|
|
|
if (!IS_ERR(in)) { |
|
udata->inlen = in->ptr_attr.len; |
|
if (uverbs_attr_ptr_is_inline(in)) |
|
udata->inbuf = |
|
&pbundle->user_attrs[in->ptr_attr.uattr_idx] |
|
.data; |
|
else |
|
udata->inbuf = u64_to_user_ptr(in->ptr_attr.data); |
|
} else { |
|
udata->inbuf = NULL; |
|
udata->inlen = 0; |
|
} |
|
|
|
if (!IS_ERR(out)) { |
|
udata->outbuf = u64_to_user_ptr(out->ptr_attr.data); |
|
udata->outlen = out->ptr_attr.len; |
|
} else { |
|
udata->outbuf = NULL; |
|
udata->outlen = 0; |
|
} |
|
} |
|
|
|
int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx, |
|
const void *from, size_t size) |
|
{ |
|
const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); |
|
size_t min_size; |
|
|
|
if (IS_ERR(attr)) |
|
return PTR_ERR(attr); |
|
|
|
min_size = min_t(size_t, attr->ptr_attr.len, size); |
|
if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size)) |
|
return -EFAULT; |
|
|
|
return uverbs_set_output(bundle, attr); |
|
} |
|
EXPORT_SYMBOL(uverbs_copy_to); |
|
|
|
|
|
/* |
|
* This is only used if the caller has directly used copy_to_use to write the |
|
* data. It signals to user space that the buffer is filled in. |
|
*/ |
|
int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx) |
|
{ |
|
const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); |
|
|
|
if (IS_ERR(attr)) |
|
return PTR_ERR(attr); |
|
|
|
return uverbs_set_output(bundle, attr); |
|
} |
|
|
|
int _uverbs_get_const(s64 *to, const struct uverbs_attr_bundle *attrs_bundle, |
|
size_t idx, s64 lower_bound, u64 upper_bound, |
|
s64 *def_val) |
|
{ |
|
const struct uverbs_attr *attr; |
|
|
|
attr = uverbs_attr_get(attrs_bundle, idx); |
|
if (IS_ERR(attr)) { |
|
if ((PTR_ERR(attr) != -ENOENT) || !def_val) |
|
return PTR_ERR(attr); |
|
|
|
*to = *def_val; |
|
} else { |
|
*to = attr->ptr_attr.data; |
|
} |
|
|
|
if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound)) |
|
return -EINVAL; |
|
|
|
return 0; |
|
} |
|
EXPORT_SYMBOL(_uverbs_get_const); |
|
|
|
int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle, |
|
size_t idx, const void *from, size_t size) |
|
{ |
|
const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); |
|
|
|
if (IS_ERR(attr)) |
|
return PTR_ERR(attr); |
|
|
|
if (size < attr->ptr_attr.len) { |
|
if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size, |
|
attr->ptr_attr.len - size)) |
|
return -EFAULT; |
|
} |
|
return uverbs_copy_to(bundle, idx, from, size); |
|
} |
|
EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero); |
|
|
|
/* Once called an abort will call through to the type's destroy_hw() */ |
|
void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle, |
|
u16 idx) |
|
{ |
|
struct bundle_priv *pbundle = |
|
container_of(bundle, struct bundle_priv, bundle); |
|
|
|
__set_bit(uapi_bkey_attr(uapi_key_attr(idx)), |
|
pbundle->uobj_hw_obj_valid); |
|
} |
|
EXPORT_SYMBOL(uverbs_finalize_uobj_create);
|
|
|