850 lines · c
1/*2 * Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.3 *4 * This software is available to you under a choice of one of two5 * licenses. You may choose to be licensed under the terms of the GNU6 * General Public License (GPL) Version 2, available from the file7 * COPYING in the main directory of this source tree, or the8 * OpenIB.org BSD license below:9 *10 * Redistribution and use in source and binary forms, with or11 * without modification, are permitted provided that the following12 * conditions are met:13 *14 * - Redistributions of source code must retain the above15 * copyright notice, this list of conditions and the following16 * disclaimer.17 *18 * - Redistributions in binary form must reproduce the above19 * copyright notice, this list of conditions and the following20 * disclaimer in the documentation and/or other materials21 * provided with the distribution.22 *23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE30 * SOFTWARE.31 */32 33#include <rdma/rdma_user_ioctl.h>34#include <rdma/uverbs_ioctl.h>35#include "rdma_core.h"36#include "uverbs.h"37 38struct bundle_alloc_head {39 struct_group_tagged(bundle_alloc_head_hdr, hdr,40 struct bundle_alloc_head *next;41 );42 u8 data[];43};44 45struct bundle_priv {46 /* Must be first */47 struct bundle_alloc_head_hdr alloc_head;48 struct bundle_alloc_head *allocated_mem;49 size_t internal_avail;50 size_t internal_used;51 52 struct radix_tree_root *radix;53 const struct uverbs_api_ioctl_method *method_elm;54 void __rcu **radix_slots;55 unsigned long radix_slots_len;56 u32 method_key;57 58 struct ib_uverbs_attr __user *user_attrs;59 struct ib_uverbs_attr *uattrs;60 61 DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);62 DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);63 DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);64 65 /*66 * Must be last. bundle ends in a flex array which overlaps67 * internal_buffer.68 */69 struct uverbs_attr_bundle_hdr bundle;70 u64 internal_buffer[32];71};72 73/*74 * Each method has an absolute minimum amount of memory it needs to allocate,75 * precompute that amount and determine if the onstack memory can be used or76 * if allocation is need.77 */78void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,79 unsigned int num_attrs)80{81 struct bundle_priv *pbundle;82 struct uverbs_attr_bundle *bundle;83 size_t bundle_size =84 offsetof(struct bundle_priv, internal_buffer) +85 sizeof(*bundle->attrs) * method_elm->key_bitmap_len +86 sizeof(*pbundle->uattrs) * num_attrs;87 88 method_elm->use_stack = bundle_size <= sizeof(*pbundle);89 method_elm->bundle_size =90 ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer));91 92 /* Do not want order-2 allocations for this. */93 WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);94}95 96/**97 * _uverbs_alloc() - Quickly allocate memory for use with a bundle98 * @bundle: The bundle99 * @size: Number of bytes to allocate100 * @flags: Allocator flags101 *102 * The bundle allocator is intended for allocations that are connected with103 * processing the system call related to the bundle. The allocated memory is104 * always freed once the system call completes, and cannot be freed any other105 * way.106 *107 * This tries to use a small pool of pre-allocated memory for performance.108 */109__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,110 gfp_t flags)111{112 struct bundle_priv *pbundle =113 container_of(&bundle->hdr, struct bundle_priv, bundle);114 size_t new_used;115 void *res;116 117 if (check_add_overflow(size, pbundle->internal_used, &new_used))118 return ERR_PTR(-EOVERFLOW);119 120 if (new_used > pbundle->internal_avail) {121 struct bundle_alloc_head *buf;122 123 buf = kvmalloc(struct_size(buf, data, size), flags);124 if (!buf)125 return ERR_PTR(-ENOMEM);126 buf->next = pbundle->allocated_mem;127 pbundle->allocated_mem = buf;128 return buf->data;129 }130 131 res = (void *)pbundle->internal_buffer + pbundle->internal_used;132 pbundle->internal_used =133 ALIGN(new_used, sizeof(*pbundle->internal_buffer));134 if (want_init_on_alloc(flags))135 memset(res, 0, size);136 return res;137}138EXPORT_SYMBOL(_uverbs_alloc);139 140static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,141 u16 len)142{143 if (uattr->len > sizeof_field(struct ib_uverbs_attr, data))144 return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len,145 uattr->len - len);146 147 return !memchr_inv((const void *)&uattr->data + len,148 0, uattr->len - len);149}150 151static int uverbs_set_output(const struct uverbs_attr_bundle *bundle,152 const struct uverbs_attr *attr)153{154 struct bundle_priv *pbundle =155 container_of(&bundle->hdr, struct bundle_priv, bundle);156 u16 flags;157 158 flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |159 UVERBS_ATTR_F_VALID_OUTPUT;160 if (put_user(flags,161 &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))162 return -EFAULT;163 return 0;164}165 166static int uverbs_process_idrs_array(struct bundle_priv *pbundle,167 const struct uverbs_api_attr *attr_uapi,168 struct uverbs_objs_arr_attr *attr,169 struct ib_uverbs_attr *uattr,170 u32 attr_bkey)171{172 struct uverbs_attr_bundle *bundle =173 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);174 const struct uverbs_attr_spec *spec = &attr_uapi->spec;175 size_t array_len;176 u32 *idr_vals;177 int ret = 0;178 size_t i;179 180 if (uattr->attr_data.reserved)181 return -EINVAL;182 183 if (uattr->len % sizeof(u32))184 return -EINVAL;185 186 array_len = uattr->len / sizeof(u32);187 if (array_len < spec->u2.objs_arr.min_len ||188 array_len > spec->u2.objs_arr.max_len)189 return -EINVAL;190 191 attr->uobjects =192 uverbs_alloc(bundle,193 array_size(array_len, sizeof(*attr->uobjects)));194 if (IS_ERR(attr->uobjects))195 return PTR_ERR(attr->uobjects);196 197 /*198 * Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects199 * to store idrs array and avoid additional memory allocation. The200 * idrs array is offset to the end of the uobjects array so we will be201 * able to read idr and replace with a pointer.202 */203 idr_vals = (u32 *)(attr->uobjects + array_len) - array_len;204 205 if (uattr->len > sizeof(uattr->data)) {206 ret = copy_from_user(idr_vals, u64_to_user_ptr(uattr->data),207 uattr->len);208 if (ret)209 return -EFAULT;210 } else {211 memcpy(idr_vals, &uattr->data, uattr->len);212 }213 214 for (i = 0; i != array_len; i++) {215 attr->uobjects[i] = uverbs_get_uobject_from_file(216 spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access,217 idr_vals[i], bundle);218 if (IS_ERR(attr->uobjects[i])) {219 ret = PTR_ERR(attr->uobjects[i]);220 break;221 }222 }223 224 attr->len = i;225 __set_bit(attr_bkey, pbundle->spec_finalize);226 return ret;227}228 229static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,230 struct uverbs_objs_arr_attr *attr,231 bool commit,232 struct uverbs_attr_bundle *attrs)233{234 const struct uverbs_attr_spec *spec = &attr_uapi->spec;235 size_t i;236 237 for (i = 0; i != attr->len; i++)238 uverbs_finalize_object(attr->uobjects[i],239 spec->u2.objs_arr.access, false, commit,240 attrs);241}242 243static int uverbs_process_attr(struct bundle_priv *pbundle,244 const struct uverbs_api_attr *attr_uapi,245 struct ib_uverbs_attr *uattr, u32 attr_bkey)246{247 const struct uverbs_attr_spec *spec = &attr_uapi->spec;248 struct uverbs_attr_bundle *bundle =249 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);250 struct uverbs_attr *e = &bundle->attrs[attr_bkey];251 const struct uverbs_attr_spec *val_spec = spec;252 struct uverbs_obj_attr *o_attr;253 254 switch (spec->type) {255 case UVERBS_ATTR_TYPE_ENUM_IN:256 if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems)257 return -EOPNOTSUPP;258 259 if (uattr->attr_data.enum_data.reserved)260 return -EINVAL;261 262 val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id];263 264 /* Currently we only support PTR_IN based enums */265 if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN)266 return -EOPNOTSUPP;267 268 e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id;269 fallthrough;270 case UVERBS_ATTR_TYPE_PTR_IN:271 /* Ensure that any data provided by userspace beyond the known272 * struct is zero. Userspace that knows how to use some future273 * longer struct will fail here if used with an old kernel and274 * non-zero content, making ABI compat/discovery simpler.275 */276 if (uattr->len > val_spec->u.ptr.len &&277 val_spec->zero_trailing &&278 !uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len))279 return -EOPNOTSUPP;280 281 fallthrough;282 case UVERBS_ATTR_TYPE_PTR_OUT:283 if (uattr->len < val_spec->u.ptr.min_len ||284 (!val_spec->zero_trailing &&285 uattr->len > val_spec->u.ptr.len))286 return -EINVAL;287 288 if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN &&289 uattr->attr_data.reserved)290 return -EINVAL;291 292 e->ptr_attr.uattr_idx = uattr - pbundle->uattrs;293 e->ptr_attr.len = uattr->len;294 295 if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) {296 void *p;297 298 p = uverbs_alloc(bundle, uattr->len);299 if (IS_ERR(p))300 return PTR_ERR(p);301 302 e->ptr_attr.ptr = p;303 304 if (copy_from_user(p, u64_to_user_ptr(uattr->data),305 uattr->len))306 return -EFAULT;307 } else {308 e->ptr_attr.data = uattr->data;309 }310 break;311 312 case UVERBS_ATTR_TYPE_IDR:313 case UVERBS_ATTR_TYPE_FD:314 if (uattr->attr_data.reserved)315 return -EINVAL;316 317 if (uattr->len != 0)318 return -EINVAL;319 320 o_attr = &e->obj_attr;321 o_attr->attr_elm = attr_uapi;322 323 /*324 * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and325 * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64326 * here without caring about truncation as we know that the327 * IDR implementation today rejects negative IDs328 */329 o_attr->uobject = uverbs_get_uobject_from_file(330 spec->u.obj.obj_type, spec->u.obj.access,331 uattr->data_s64, bundle);332 if (IS_ERR(o_attr->uobject))333 return PTR_ERR(o_attr->uobject);334 __set_bit(attr_bkey, pbundle->uobj_finalize);335 336 if (spec->u.obj.access == UVERBS_ACCESS_NEW) {337 unsigned int uattr_idx = uattr - pbundle->uattrs;338 s64 id = o_attr->uobject->id;339 340 /* Copy the allocated id to the user-space */341 if (put_user(id, &pbundle->user_attrs[uattr_idx].data))342 return -EFAULT;343 }344 345 break;346 347 case UVERBS_ATTR_TYPE_RAW_FD:348 if (uattr->attr_data.reserved || uattr->len != 0 ||349 uattr->data_s64 < INT_MIN || uattr->data_s64 > INT_MAX)350 return -EINVAL;351 /* _uverbs_get_const_signed() is the accessor */352 e->ptr_attr.data = uattr->data_s64;353 break;354 355 case UVERBS_ATTR_TYPE_IDRS_ARRAY:356 return uverbs_process_idrs_array(pbundle, attr_uapi,357 &e->objs_arr_attr, uattr,358 attr_bkey);359 default:360 return -EOPNOTSUPP;361 }362 363 return 0;364}365 366/*367 * We search the radix tree with the method prefix and now we want to fast368 * search the suffix bits to get a particular attribute pointer. It is not369 * totally clear to me if this breaks the radix tree encasulation or not, but370 * it uses the iter data to determine if the method iter points at the same371 * chunk that will store the attribute, if so it just derefs it directly. By372 * construction in most kernel configs the method and attrs will all fit in a373 * single radix chunk, so in most cases this will have no search. Other cases374 * this falls back to a full search.375 */376static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle,377 u32 attr_key)378{379 void __rcu **slot;380 381 if (likely(attr_key < pbundle->radix_slots_len)) {382 void *entry;383 384 slot = pbundle->radix_slots + attr_key;385 entry = rcu_dereference_raw(*slot);386 if (likely(!radix_tree_is_internal_node(entry) && entry))387 return slot;388 }389 390 return radix_tree_lookup_slot(pbundle->radix,391 pbundle->method_key | attr_key);392}393 394static int uverbs_set_attr(struct bundle_priv *pbundle,395 struct ib_uverbs_attr *uattr)396{397 u32 attr_key = uapi_key_attr(uattr->attr_id);398 u32 attr_bkey = uapi_bkey_attr(attr_key);399 const struct uverbs_api_attr *attr;400 void __rcu **slot;401 int ret;402 403 slot = uapi_get_attr_for_method(pbundle, attr_key);404 if (!slot) {405 /*406 * Kernel does not support the attribute but user-space says it407 * is mandatory408 */409 if (uattr->flags & UVERBS_ATTR_F_MANDATORY)410 return -EPROTONOSUPPORT;411 return 0;412 }413 attr = rcu_dereference_protected(*slot, true);414 415 /* Reject duplicate attributes from user-space */416 if (test_bit(attr_bkey, pbundle->bundle.attr_present))417 return -EINVAL;418 419 ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey);420 if (ret)421 return ret;422 423 __set_bit(attr_bkey, pbundle->bundle.attr_present);424 425 return 0;426}427 428static int ib_uverbs_run_method(struct bundle_priv *pbundle,429 unsigned int num_attrs)430{431 int (*handler)(struct uverbs_attr_bundle *attrs);432 struct uverbs_attr_bundle *bundle =433 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);434 size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);435 unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;436 unsigned int i;437 int ret;438 439 /* See uverbs_disassociate_api() */440 handler = srcu_dereference(441 pbundle->method_elm->handler,442 &pbundle->bundle.ufile->device->disassociate_srcu);443 if (!handler)444 return -EIO;445 446 pbundle->uattrs = uverbs_alloc(bundle, uattrs_size);447 if (IS_ERR(pbundle->uattrs))448 return PTR_ERR(pbundle->uattrs);449 if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size))450 return -EFAULT;451 452 for (i = 0; i != num_attrs; i++) {453 ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]);454 if (unlikely(ret))455 return ret;456 }457 458 /* User space did not provide all the mandatory attributes */459 if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,460 pbundle->bundle.attr_present,461 pbundle->method_elm->key_bitmap_len)))462 return -EINVAL;463 464 if (pbundle->method_elm->has_udata)465 uverbs_fill_udata(bundle, &pbundle->bundle.driver_udata,466 UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);467 else468 pbundle->bundle.driver_udata = (struct ib_udata){};469 470 if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {471 struct uverbs_obj_attr *destroy_attr = &bundle->attrs[destroy_bkey].obj_attr;472 473 ret = uobj_destroy(destroy_attr->uobject, bundle);474 if (ret)475 return ret;476 __clear_bit(destroy_bkey, pbundle->uobj_finalize);477 478 ret = handler(bundle);479 uobj_put_destroy(destroy_attr->uobject);480 } else {481 ret = handler(bundle);482 }483 484 /*485 * Until the drivers are revised to use the bundle directly we have to486 * assume that the driver wrote to its UHW_OUT and flag userspace487 * appropriately.488 */489 if (!ret && pbundle->method_elm->has_udata) {490 const struct uverbs_attr *attr =491 uverbs_attr_get(bundle, UVERBS_ATTR_UHW_OUT);492 493 if (!IS_ERR(attr))494 ret = uverbs_set_output(bundle, attr);495 }496 497 /*498 * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can499 * not invoke the method because the request is not supported. No500 * other cases should return this code.501 */502 if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT))503 return -EINVAL;504 505 return ret;506}507 508static void bundle_destroy(struct bundle_priv *pbundle, bool commit)509{510 unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;511 struct uverbs_attr_bundle *bundle =512 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);513 struct bundle_alloc_head *memblock;514 unsigned int i;515 516 /* fast path for simple uobjects */517 i = -1;518 while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len,519 i + 1)) < key_bitmap_len) {520 struct uverbs_attr *attr = &bundle->attrs[i];521 522 uverbs_finalize_object(523 attr->obj_attr.uobject,524 attr->obj_attr.attr_elm->spec.u.obj.access,525 test_bit(i, pbundle->uobj_hw_obj_valid),526 commit, bundle);527 }528 529 i = -1;530 while ((i = find_next_bit(pbundle->spec_finalize, key_bitmap_len,531 i + 1)) < key_bitmap_len) {532 struct uverbs_attr *attr = &bundle->attrs[i];533 const struct uverbs_api_attr *attr_uapi;534 void __rcu **slot;535 536 slot = uapi_get_attr_for_method(537 pbundle,538 pbundle->method_key | uapi_bkey_to_key_attr(i));539 if (WARN_ON(!slot))540 continue;541 542 attr_uapi = rcu_dereference_protected(*slot, true);543 544 if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {545 uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr,546 commit, bundle);547 }548 }549 550 for (memblock = pbundle->allocated_mem; memblock;) {551 struct bundle_alloc_head *tmp = memblock;552 553 memblock = memblock->next;554 kvfree(tmp);555 }556}557 558static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,559 struct ib_uverbs_ioctl_hdr *hdr,560 struct ib_uverbs_attr __user *user_attrs)561{562 const struct uverbs_api_ioctl_method *method_elm;563 struct uverbs_api *uapi = ufile->device->uapi;564 struct radix_tree_iter attrs_iter;565 struct bundle_priv *pbundle;566 struct bundle_priv onstack;567 void __rcu **slot;568 int ret;569 570 if (unlikely(hdr->driver_id != uapi->driver_id))571 return -EINVAL;572 573 slot = radix_tree_iter_lookup(574 &uapi->radix, &attrs_iter,575 uapi_key_obj(hdr->object_id) |576 uapi_key_ioctl_method(hdr->method_id));577 if (unlikely(!slot))578 return -EPROTONOSUPPORT;579 method_elm = rcu_dereference_protected(*slot, true);580 581 if (!method_elm->use_stack) {582 pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL);583 if (!pbundle)584 return -ENOMEM;585 pbundle->internal_avail =586 method_elm->bundle_size -587 offsetof(struct bundle_priv, internal_buffer);588 pbundle->alloc_head.next = NULL;589 pbundle->allocated_mem = container_of(&pbundle->alloc_head,590 struct bundle_alloc_head, hdr);591 } else {592 pbundle = &onstack;593 pbundle->internal_avail = sizeof(pbundle->internal_buffer);594 pbundle->allocated_mem = NULL;595 }596 597 /* Space for the pbundle->bundle.attrs flex array */598 pbundle->method_elm = method_elm;599 pbundle->method_key = attrs_iter.index;600 pbundle->bundle.ufile = ufile;601 pbundle->bundle.context = NULL; /* only valid if bundle has uobject */602 pbundle->radix = &uapi->radix;603 pbundle->radix_slots = slot;604 pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter);605 pbundle->user_attrs = user_attrs;606 607 pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *608 sizeof(*container_of(&pbundle->bundle,609 struct uverbs_attr_bundle, hdr)->attrs),610 sizeof(*pbundle->internal_buffer));611 memset(pbundle->bundle.attr_present, 0,612 sizeof(pbundle->bundle.attr_present));613 memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));614 memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize));615 memset(pbundle->uobj_hw_obj_valid, 0,616 sizeof(pbundle->uobj_hw_obj_valid));617 618 ret = ib_uverbs_run_method(pbundle, hdr->num_attrs);619 bundle_destroy(pbundle, ret == 0);620 return ret;621}622 623long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)624{625 struct ib_uverbs_file *file = filp->private_data;626 struct ib_uverbs_ioctl_hdr __user *user_hdr =627 (struct ib_uverbs_ioctl_hdr __user *)arg;628 struct ib_uverbs_ioctl_hdr hdr;629 int srcu_key;630 int err;631 632 if (unlikely(cmd != RDMA_VERBS_IOCTL))633 return -ENOIOCTLCMD;634 635 err = copy_from_user(&hdr, user_hdr, sizeof(hdr));636 if (err)637 return -EFAULT;638 639 if (hdr.length > PAGE_SIZE ||640 hdr.length != struct_size(&hdr, attrs, hdr.num_attrs))641 return -EINVAL;642 643 if (hdr.reserved1 || hdr.reserved2)644 return -EPROTONOSUPPORT;645 646 srcu_key = srcu_read_lock(&file->device->disassociate_srcu);647 err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs);648 srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);649 return err;650}651 652int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,653 size_t idx, u64 allowed_bits)654{655 const struct uverbs_attr *attr;656 u64 flags;657 658 attr = uverbs_attr_get(attrs_bundle, idx);659 /* Missing attribute means 0 flags */660 if (IS_ERR(attr)) {661 *to = 0;662 return 0;663 }664 665 /*666 * New userspace code should use 8 bytes to pass flags, but we667 * transparently support old userspaces that were using 4 bytes as668 * well.669 */670 if (attr->ptr_attr.len == 8)671 flags = attr->ptr_attr.data;672 else if (attr->ptr_attr.len == 4)673 flags = *(u32 *)&attr->ptr_attr.data;674 else675 return -EINVAL;676 677 if (flags & ~allowed_bits)678 return -EINVAL;679 680 *to = flags;681 return 0;682}683EXPORT_SYMBOL(uverbs_get_flags64);684 685int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,686 size_t idx, u64 allowed_bits)687{688 u64 flags;689 int ret;690 691 ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);692 if (ret)693 return ret;694 695 if (flags > U32_MAX)696 return -EINVAL;697 *to = flags;698 699 return 0;700}701EXPORT_SYMBOL(uverbs_get_flags32);702 703/*704 * Fill a ib_udata struct (core or uhw) using the given attribute IDs.705 * This is primarily used to convert the UVERBS_ATTR_UHW() into the706 * ib_udata format used by the drivers.707 */708void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,709 struct ib_udata *udata, unsigned int attr_in,710 unsigned int attr_out)711{712 struct bundle_priv *pbundle =713 container_of(&bundle->hdr, struct bundle_priv, bundle);714 struct uverbs_attr_bundle *bundle_aux =715 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);716 const struct uverbs_attr *in =717 uverbs_attr_get(bundle_aux, attr_in);718 const struct uverbs_attr *out =719 uverbs_attr_get(bundle_aux, attr_out);720 721 if (!IS_ERR(in)) {722 udata->inlen = in->ptr_attr.len;723 if (uverbs_attr_ptr_is_inline(in))724 udata->inbuf =725 &pbundle->user_attrs[in->ptr_attr.uattr_idx]726 .data;727 else728 udata->inbuf = u64_to_user_ptr(in->ptr_attr.data);729 } else {730 udata->inbuf = NULL;731 udata->inlen = 0;732 }733 734 if (!IS_ERR(out)) {735 udata->outbuf = u64_to_user_ptr(out->ptr_attr.data);736 udata->outlen = out->ptr_attr.len;737 } else {738 udata->outbuf = NULL;739 udata->outlen = 0;740 }741}742 743int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,744 const void *from, size_t size)745{746 const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);747 size_t min_size;748 749 if (IS_ERR(attr))750 return PTR_ERR(attr);751 752 min_size = min_t(size_t, attr->ptr_attr.len, size);753 if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))754 return -EFAULT;755 756 return uverbs_set_output(bundle, attr);757}758EXPORT_SYMBOL(uverbs_copy_to);759 760 761/*762 * This is only used if the caller has directly used copy_to_use to write the763 * data. It signals to user space that the buffer is filled in.764 */765int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx)766{767 const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);768 769 if (IS_ERR(attr))770 return PTR_ERR(attr);771 772 return uverbs_set_output(bundle, attr);773}774 775int _uverbs_get_const_signed(s64 *to,776 const struct uverbs_attr_bundle *attrs_bundle,777 size_t idx, s64 lower_bound, u64 upper_bound,778 s64 *def_val)779{780 const struct uverbs_attr *attr;781 782 attr = uverbs_attr_get(attrs_bundle, idx);783 if (IS_ERR(attr)) {784 if ((PTR_ERR(attr) != -ENOENT) || !def_val)785 return PTR_ERR(attr);786 787 *to = *def_val;788 } else {789 *to = attr->ptr_attr.data;790 }791 792 if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound))793 return -EINVAL;794 795 return 0;796}797EXPORT_SYMBOL(_uverbs_get_const_signed);798 799int _uverbs_get_const_unsigned(u64 *to,800 const struct uverbs_attr_bundle *attrs_bundle,801 size_t idx, u64 upper_bound, u64 *def_val)802{803 const struct uverbs_attr *attr;804 805 attr = uverbs_attr_get(attrs_bundle, idx);806 if (IS_ERR(attr)) {807 if ((PTR_ERR(attr) != -ENOENT) || !def_val)808 return PTR_ERR(attr);809 810 *to = *def_val;811 } else {812 *to = attr->ptr_attr.data;813 }814 815 if (*to > upper_bound)816 return -EINVAL;817 818 return 0;819}820EXPORT_SYMBOL(_uverbs_get_const_unsigned);821 822int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,823 size_t idx, const void *from, size_t size)824{825 const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);826 827 if (IS_ERR(attr))828 return PTR_ERR(attr);829 830 if (size < attr->ptr_attr.len) {831 if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size,832 attr->ptr_attr.len - size))833 return -EFAULT;834 }835 return uverbs_copy_to(bundle, idx, from, size);836}837EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);838 839/* Once called an abort will call through to the type's destroy_hw() */840void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,841 u16 idx)842{843 struct bundle_priv *pbundle =844 container_of(&bundle->hdr, struct bundle_priv, bundle);845 846 __set_bit(uapi_bkey_attr(uapi_key_attr(idx)),847 pbundle->uobj_hw_obj_valid);848}849EXPORT_SYMBOL(uverbs_finalize_uobj_create);850