735 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB2/*3 * Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.4 */5#include <rdma/uverbs_ioctl.h>6#include <rdma/rdma_user_ioctl.h>7#include <linux/bitops.h>8#include "rdma_core.h"9#include "uverbs.h"10 11static int ib_uverbs_notsupp(struct uverbs_attr_bundle *attrs)12{13 return -EOPNOTSUPP;14}15 16static void *uapi_add_elm(struct uverbs_api *uapi, u32 key, size_t alloc_size)17{18 void *elm;19 int rc;20 21 if (key == UVERBS_API_KEY_ERR)22 return ERR_PTR(-EOVERFLOW);23 24 elm = kzalloc(alloc_size, GFP_KERNEL);25 if (!elm)26 return ERR_PTR(-ENOMEM);27 rc = radix_tree_insert(&uapi->radix, key, elm);28 if (rc) {29 kfree(elm);30 return ERR_PTR(rc);31 }32 33 return elm;34}35 36static void *uapi_add_get_elm(struct uverbs_api *uapi, u32 key,37 size_t alloc_size, bool *exists)38{39 void *elm;40 41 elm = uapi_add_elm(uapi, key, alloc_size);42 if (!IS_ERR(elm)) {43 *exists = false;44 return elm;45 }46 47 if (elm != ERR_PTR(-EEXIST))48 return elm;49 50 elm = radix_tree_lookup(&uapi->radix, key);51 if (WARN_ON(!elm))52 return ERR_PTR(-EINVAL);53 *exists = true;54 return elm;55}56 57static int uapi_create_write(struct uverbs_api *uapi,58 struct ib_device *ibdev,59 const struct uapi_definition *def,60 u32 obj_key,61 u32 *cur_method_key)62{63 struct uverbs_api_write_method *method_elm;64 u32 method_key = obj_key;65 bool exists;66 67 if (def->write.is_ex)68 method_key |= uapi_key_write_ex_method(def->write.command_num);69 else70 method_key |= uapi_key_write_method(def->write.command_num);71 72 method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),73 &exists);74 if (IS_ERR(method_elm))75 return PTR_ERR(method_elm);76 77 if (WARN_ON(exists && (def->write.is_ex != method_elm->is_ex)))78 return -EINVAL;79 80 method_elm->is_ex = def->write.is_ex;81 method_elm->handler = def->func_write;82 if (!def->write.is_ex)83 method_elm->disabled = !(ibdev->uverbs_cmd_mask &84 BIT_ULL(def->write.command_num));85 86 if (!def->write.is_ex && def->func_write) {87 method_elm->has_udata = def->write.has_udata;88 method_elm->has_resp = def->write.has_resp;89 method_elm->req_size = def->write.req_size;90 method_elm->resp_size = def->write.resp_size;91 }92 93 *cur_method_key = method_key;94 return 0;95}96 97static int uapi_merge_method(struct uverbs_api *uapi,98 struct uverbs_api_object *obj_elm, u32 obj_key,99 const struct uverbs_method_def *method,100 bool is_driver)101{102 u32 method_key = obj_key | uapi_key_ioctl_method(method->id);103 struct uverbs_api_ioctl_method *method_elm;104 unsigned int i;105 bool exists;106 107 if (!method->attrs)108 return 0;109 110 method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),111 &exists);112 if (IS_ERR(method_elm))113 return PTR_ERR(method_elm);114 if (exists) {115 /*116 * This occurs when a driver uses ADD_UVERBS_ATTRIBUTES_SIMPLE117 */118 if (WARN_ON(method->handler))119 return -EINVAL;120 } else {121 WARN_ON(!method->handler);122 rcu_assign_pointer(method_elm->handler, method->handler);123 if (method->handler != uverbs_destroy_def_handler)124 method_elm->driver_method = is_driver;125 }126 127 for (i = 0; i != method->num_attrs; i++) {128 const struct uverbs_attr_def *attr = (*method->attrs)[i];129 struct uverbs_api_attr *attr_slot;130 131 if (!attr)132 continue;133 134 /*135 * ENUM_IN contains the 'ids' pointer to the driver's .rodata,136 * so if it is specified by a driver then it always makes this137 * into a driver method.138 */139 if (attr->attr.type == UVERBS_ATTR_TYPE_ENUM_IN)140 method_elm->driver_method |= is_driver;141 142 /*143 * Like other uobject based things we only support a single144 * uobject being NEW'd or DESTROY'd145 */146 if (attr->attr.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {147 u8 access = attr->attr.u2.objs_arr.access;148 149 if (WARN_ON(access == UVERBS_ACCESS_NEW ||150 access == UVERBS_ACCESS_DESTROY))151 return -EINVAL;152 }153 154 attr_slot =155 uapi_add_elm(uapi, method_key | uapi_key_attr(attr->id),156 sizeof(*attr_slot));157 /* Attributes are not allowed to be modified by drivers */158 if (IS_ERR(attr_slot))159 return PTR_ERR(attr_slot);160 161 attr_slot->spec = attr->attr;162 }163 164 return 0;165}166 167static int uapi_merge_obj_tree(struct uverbs_api *uapi,168 const struct uverbs_object_def *obj,169 bool is_driver)170{171 struct uverbs_api_object *obj_elm;172 unsigned int i;173 u32 obj_key;174 bool exists;175 int rc;176 177 obj_key = uapi_key_obj(obj->id);178 obj_elm = uapi_add_get_elm(uapi, obj_key, sizeof(*obj_elm), &exists);179 if (IS_ERR(obj_elm))180 return PTR_ERR(obj_elm);181 182 if (obj->type_attrs) {183 if (WARN_ON(obj_elm->type_attrs))184 return -EINVAL;185 186 obj_elm->id = obj->id;187 obj_elm->type_attrs = obj->type_attrs;188 obj_elm->type_class = obj->type_attrs->type_class;189 /*190 * Today drivers are only permitted to use idr_class and191 * fd_class types. We can revoke the IDR types during192 * disassociation, and the FD types require the driver to use193 * struct file_operations.owner to prevent the driver module194 * code from unloading while the file is open. This provides195 * enough safety that uverbs_uobject_fd_release() will196 * continue to work. Drivers using FD are responsible to197 * handle disassociation of the device on their own.198 */199 if (WARN_ON(is_driver &&200 obj->type_attrs->type_class != &uverbs_idr_class &&201 obj->type_attrs->type_class != &uverbs_fd_class))202 return -EINVAL;203 }204 205 if (!obj->methods)206 return 0;207 208 for (i = 0; i != obj->num_methods; i++) {209 const struct uverbs_method_def *method = (*obj->methods)[i];210 211 if (!method)212 continue;213 214 rc = uapi_merge_method(uapi, obj_elm, obj_key, method,215 is_driver);216 if (rc)217 return rc;218 }219 220 return 0;221}222 223static int uapi_disable_elm(struct uverbs_api *uapi,224 const struct uapi_definition *def,225 u32 obj_key,226 u32 method_key)227{228 bool exists;229 230 if (def->scope == UAPI_SCOPE_OBJECT) {231 struct uverbs_api_object *obj_elm;232 233 obj_elm = uapi_add_get_elm(234 uapi, obj_key, sizeof(*obj_elm), &exists);235 if (IS_ERR(obj_elm))236 return PTR_ERR(obj_elm);237 obj_elm->disabled = 1;238 return 0;239 }240 241 if (def->scope == UAPI_SCOPE_METHOD &&242 uapi_key_is_ioctl_method(method_key)) {243 struct uverbs_api_ioctl_method *method_elm;244 245 method_elm = uapi_add_get_elm(uapi, method_key,246 sizeof(*method_elm), &exists);247 if (IS_ERR(method_elm))248 return PTR_ERR(method_elm);249 method_elm->disabled = 1;250 return 0;251 }252 253 if (def->scope == UAPI_SCOPE_METHOD &&254 (uapi_key_is_write_method(method_key) ||255 uapi_key_is_write_ex_method(method_key))) {256 struct uverbs_api_write_method *write_elm;257 258 write_elm = uapi_add_get_elm(uapi, method_key,259 sizeof(*write_elm), &exists);260 if (IS_ERR(write_elm))261 return PTR_ERR(write_elm);262 write_elm->disabled = 1;263 return 0;264 }265 266 WARN_ON(true);267 return -EINVAL;268}269 270static int uapi_merge_def(struct uverbs_api *uapi, struct ib_device *ibdev,271 const struct uapi_definition *def_list,272 bool is_driver)273{274 const struct uapi_definition *def = def_list;275 u32 cur_obj_key = UVERBS_API_KEY_ERR;276 u32 cur_method_key = UVERBS_API_KEY_ERR;277 bool exists;278 int rc;279 280 if (!def_list)281 return 0;282 283 for (;; def++) {284 switch ((enum uapi_definition_kind)def->kind) {285 case UAPI_DEF_CHAIN:286 rc = uapi_merge_def(uapi, ibdev, def->chain, is_driver);287 if (rc)288 return rc;289 continue;290 291 case UAPI_DEF_CHAIN_OBJ_TREE:292 if (WARN_ON(def->object_start.object_id !=293 def->chain_obj_tree->id))294 return -EINVAL;295 296 cur_obj_key = uapi_key_obj(def->object_start.object_id);297 rc = uapi_merge_obj_tree(uapi, def->chain_obj_tree,298 is_driver);299 if (rc)300 return rc;301 continue;302 303 case UAPI_DEF_END:304 return 0;305 306 case UAPI_DEF_IS_SUPPORTED_DEV_FN: {307 void **ibdev_fn =308 (void *)(&ibdev->ops) + def->needs_fn_offset;309 310 if (*ibdev_fn)311 continue;312 rc = uapi_disable_elm(313 uapi, def, cur_obj_key, cur_method_key);314 if (rc)315 return rc;316 continue;317 }318 319 case UAPI_DEF_IS_SUPPORTED_FUNC:320 if (def->func_is_supported(ibdev))321 continue;322 rc = uapi_disable_elm(323 uapi, def, cur_obj_key, cur_method_key);324 if (rc)325 return rc;326 continue;327 328 case UAPI_DEF_OBJECT_START: {329 struct uverbs_api_object *obj_elm;330 331 cur_obj_key = uapi_key_obj(def->object_start.object_id);332 obj_elm = uapi_add_get_elm(uapi, cur_obj_key,333 sizeof(*obj_elm), &exists);334 if (IS_ERR(obj_elm))335 return PTR_ERR(obj_elm);336 continue;337 }338 339 case UAPI_DEF_WRITE:340 rc = uapi_create_write(341 uapi, ibdev, def, cur_obj_key, &cur_method_key);342 if (rc)343 return rc;344 continue;345 }346 WARN_ON(true);347 return -EINVAL;348 }349}350 351static int352uapi_finalize_ioctl_method(struct uverbs_api *uapi,353 struct uverbs_api_ioctl_method *method_elm,354 u32 method_key)355{356 struct radix_tree_iter iter;357 unsigned int num_attrs = 0;358 unsigned int max_bkey = 0;359 bool single_uobj = false;360 void __rcu **slot;361 362 method_elm->destroy_bkey = UVERBS_API_ATTR_BKEY_LEN;363 radix_tree_for_each_slot (slot, &uapi->radix, &iter,364 uapi_key_attrs_start(method_key)) {365 struct uverbs_api_attr *elm =366 rcu_dereference_protected(*slot, true);367 u32 attr_key = iter.index & UVERBS_API_ATTR_KEY_MASK;368 u32 attr_bkey = uapi_bkey_attr(attr_key);369 u8 type = elm->spec.type;370 371 if (uapi_key_attr_to_ioctl_method(iter.index) !=372 uapi_key_attr_to_ioctl_method(method_key))373 break;374 375 if (elm->spec.mandatory)376 __set_bit(attr_bkey, method_elm->attr_mandatory);377 378 if (elm->spec.is_udata)379 method_elm->has_udata = true;380 381 if (type == UVERBS_ATTR_TYPE_IDR ||382 type == UVERBS_ATTR_TYPE_FD) {383 u8 access = elm->spec.u.obj.access;384 385 /*386 * Verbs specs may only have one NEW/DESTROY, we don't387 * have the infrastructure to abort multiple NEW's or388 * cope with multiple DESTROY failure.389 */390 if (access == UVERBS_ACCESS_NEW ||391 access == UVERBS_ACCESS_DESTROY) {392 if (WARN_ON(single_uobj))393 return -EINVAL;394 395 single_uobj = true;396 if (WARN_ON(!elm->spec.mandatory))397 return -EINVAL;398 }399 400 if (access == UVERBS_ACCESS_DESTROY)401 method_elm->destroy_bkey = attr_bkey;402 }403 404 max_bkey = max(max_bkey, attr_bkey);405 num_attrs++;406 }407 408 method_elm->key_bitmap_len = max_bkey + 1;409 WARN_ON(method_elm->key_bitmap_len > UVERBS_API_ATTR_BKEY_LEN);410 411 uapi_compute_bundle_size(method_elm, num_attrs);412 return 0;413}414 415static int uapi_finalize(struct uverbs_api *uapi)416{417 const struct uverbs_api_write_method **data;418 unsigned long max_write_ex = 0;419 unsigned long max_write = 0;420 struct radix_tree_iter iter;421 void __rcu **slot;422 int rc;423 int i;424 425 radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {426 struct uverbs_api_ioctl_method *method_elm =427 rcu_dereference_protected(*slot, true);428 429 if (uapi_key_is_ioctl_method(iter.index)) {430 rc = uapi_finalize_ioctl_method(uapi, method_elm,431 iter.index);432 if (rc)433 return rc;434 }435 436 if (uapi_key_is_write_method(iter.index))437 max_write = max(max_write,438 iter.index & UVERBS_API_ATTR_KEY_MASK);439 if (uapi_key_is_write_ex_method(iter.index))440 max_write_ex =441 max(max_write_ex,442 iter.index & UVERBS_API_ATTR_KEY_MASK);443 }444 445 uapi->notsupp_method.handler = ib_uverbs_notsupp;446 uapi->num_write = max_write + 1;447 uapi->num_write_ex = max_write_ex + 1;448 data = kmalloc_array(uapi->num_write + uapi->num_write_ex,449 sizeof(*uapi->write_methods), GFP_KERNEL);450 if (!data)451 return -ENOMEM;452 453 for (i = 0; i != uapi->num_write + uapi->num_write_ex; i++)454 data[i] = &uapi->notsupp_method;455 uapi->write_methods = data;456 uapi->write_ex_methods = data + uapi->num_write;457 458 radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {459 if (uapi_key_is_write_method(iter.index))460 uapi->write_methods[iter.index &461 UVERBS_API_ATTR_KEY_MASK] =462 rcu_dereference_protected(*slot, true);463 if (uapi_key_is_write_ex_method(iter.index))464 uapi->write_ex_methods[iter.index &465 UVERBS_API_ATTR_KEY_MASK] =466 rcu_dereference_protected(*slot, true);467 }468 469 return 0;470}471 472static void uapi_remove_range(struct uverbs_api *uapi, u32 start, u32 last)473{474 struct radix_tree_iter iter;475 void __rcu **slot;476 477 radix_tree_for_each_slot (slot, &uapi->radix, &iter, start) {478 if (iter.index > last)479 return;480 kfree(rcu_dereference_protected(*slot, true));481 radix_tree_iter_delete(&uapi->radix, &iter, slot);482 }483}484 485static void uapi_remove_object(struct uverbs_api *uapi, u32 obj_key)486{487 uapi_remove_range(uapi, obj_key,488 obj_key | UVERBS_API_METHOD_KEY_MASK |489 UVERBS_API_ATTR_KEY_MASK);490}491 492static void uapi_remove_method(struct uverbs_api *uapi, u32 method_key)493{494 uapi_remove_range(uapi, method_key,495 method_key | UVERBS_API_ATTR_KEY_MASK);496}497 498 499static u32 uapi_get_obj_id(struct uverbs_attr_spec *spec)500{501 if (spec->type == UVERBS_ATTR_TYPE_IDR ||502 spec->type == UVERBS_ATTR_TYPE_FD)503 return spec->u.obj.obj_type;504 if (spec->type == UVERBS_ATTR_TYPE_IDRS_ARRAY)505 return spec->u2.objs_arr.obj_type;506 return UVERBS_API_KEY_ERR;507}508 509static void uapi_key_okay(u32 key)510{511 unsigned int count = 0;512 513 if (uapi_key_is_object(key))514 count++;515 if (uapi_key_is_ioctl_method(key))516 count++;517 if (uapi_key_is_write_method(key))518 count++;519 if (uapi_key_is_write_ex_method(key))520 count++;521 if (uapi_key_is_attr(key))522 count++;523 WARN(count != 1, "Bad count %u key=%x", count, key);524}525 526static void uapi_finalize_disable(struct uverbs_api *uapi)527{528 struct radix_tree_iter iter;529 u32 starting_key = 0;530 bool scan_again = false;531 void __rcu **slot;532 533again:534 radix_tree_for_each_slot (slot, &uapi->radix, &iter, starting_key) {535 uapi_key_okay(iter.index);536 537 if (uapi_key_is_object(iter.index)) {538 struct uverbs_api_object *obj_elm =539 rcu_dereference_protected(*slot, true);540 541 if (obj_elm->disabled) {542 /* Have to check all the attrs again */543 scan_again = true;544 starting_key = iter.index;545 uapi_remove_object(uapi, iter.index);546 goto again;547 }548 continue;549 }550 551 if (uapi_key_is_ioctl_method(iter.index)) {552 struct uverbs_api_ioctl_method *method_elm =553 rcu_dereference_protected(*slot, true);554 555 if (method_elm->disabled) {556 starting_key = iter.index;557 uapi_remove_method(uapi, iter.index);558 goto again;559 }560 continue;561 }562 563 if (uapi_key_is_write_method(iter.index) ||564 uapi_key_is_write_ex_method(iter.index)) {565 struct uverbs_api_write_method *method_elm =566 rcu_dereference_protected(*slot, true);567 568 if (method_elm->disabled) {569 kfree(method_elm);570 radix_tree_iter_delete(&uapi->radix, &iter, slot);571 }572 continue;573 }574 575 if (uapi_key_is_attr(iter.index)) {576 struct uverbs_api_attr *attr_elm =577 rcu_dereference_protected(*slot, true);578 const struct uverbs_api_object *tmp_obj;579 u32 obj_key;580 581 /*582 * If the method has a mandatory object handle583 * attribute which relies on an object which is not584 * present then the entire method is uncallable.585 */586 if (!attr_elm->spec.mandatory)587 continue;588 obj_key = uapi_get_obj_id(&attr_elm->spec);589 if (obj_key == UVERBS_API_KEY_ERR)590 continue;591 tmp_obj = uapi_get_object(uapi, obj_key);592 if (IS_ERR(tmp_obj)) {593 if (PTR_ERR(tmp_obj) == -ENOMSG)594 continue;595 } else {596 if (!tmp_obj->disabled)597 continue;598 }599 600 starting_key = iter.index;601 uapi_remove_method(602 uapi,603 iter.index & (UVERBS_API_OBJ_KEY_MASK |604 UVERBS_API_METHOD_KEY_MASK));605 goto again;606 }607 608 WARN_ON(false);609 }610 611 if (!scan_again)612 return;613 scan_again = false;614 starting_key = 0;615 goto again;616}617 618void uverbs_destroy_api(struct uverbs_api *uapi)619{620 if (!uapi)621 return;622 623 uapi_remove_range(uapi, 0, U32_MAX);624 kfree(uapi->write_methods);625 kfree(uapi);626}627 628static const struct uapi_definition uverbs_core_api[] = {629 UAPI_DEF_CHAIN(uverbs_def_obj_async_fd),630 UAPI_DEF_CHAIN(uverbs_def_obj_counters),631 UAPI_DEF_CHAIN(uverbs_def_obj_cq),632 UAPI_DEF_CHAIN(uverbs_def_obj_device),633 UAPI_DEF_CHAIN(uverbs_def_obj_dm),634 UAPI_DEF_CHAIN(uverbs_def_obj_flow_action),635 UAPI_DEF_CHAIN(uverbs_def_obj_intf),636 UAPI_DEF_CHAIN(uverbs_def_obj_mr),637 UAPI_DEF_CHAIN(uverbs_def_obj_qp),638 UAPI_DEF_CHAIN(uverbs_def_obj_srq),639 UAPI_DEF_CHAIN(uverbs_def_obj_wq),640 UAPI_DEF_CHAIN(uverbs_def_write_intf),641 {},642};643 644struct uverbs_api *uverbs_alloc_api(struct ib_device *ibdev)645{646 struct uverbs_api *uapi;647 int rc;648 649 uapi = kzalloc(sizeof(*uapi), GFP_KERNEL);650 if (!uapi)651 return ERR_PTR(-ENOMEM);652 653 INIT_RADIX_TREE(&uapi->radix, GFP_KERNEL);654 uapi->driver_id = ibdev->ops.driver_id;655 656 rc = uapi_merge_def(uapi, ibdev, uverbs_core_api, false);657 if (rc)658 goto err;659 rc = uapi_merge_def(uapi, ibdev, ibdev->driver_def, true);660 if (rc)661 goto err;662 663 uapi_finalize_disable(uapi);664 rc = uapi_finalize(uapi);665 if (rc)666 goto err;667 668 return uapi;669err:670 if (rc != -ENOMEM)671 dev_err(&ibdev->dev,672 "Setup of uverbs_api failed, kernel parsing tree description is not valid (%d)??\n",673 rc);674 675 uverbs_destroy_api(uapi);676 return ERR_PTR(rc);677}678 679/*680 * The pre version is done before destroying the HW objects, it only blocks681 * off method access. All methods that require the ib_dev or the module data682 * must test one of these assignments prior to continuing.683 */684void uverbs_disassociate_api_pre(struct ib_uverbs_device *uverbs_dev)685{686 struct uverbs_api *uapi = uverbs_dev->uapi;687 struct radix_tree_iter iter;688 void __rcu **slot;689 690 rcu_assign_pointer(uverbs_dev->ib_dev, NULL);691 692 radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {693 if (uapi_key_is_ioctl_method(iter.index)) {694 struct uverbs_api_ioctl_method *method_elm =695 rcu_dereference_protected(*slot, true);696 697 if (method_elm->driver_method)698 rcu_assign_pointer(method_elm->handler, NULL);699 }700 }701 702 synchronize_srcu(&uverbs_dev->disassociate_srcu);703}704 705/*706 * Called when a driver disassociates from the ib_uverbs_device. The707 * assumption is that the driver module will unload after. Replace everything708 * related to the driver with NULL as a safety measure.709 */710void uverbs_disassociate_api(struct uverbs_api *uapi)711{712 struct radix_tree_iter iter;713 void __rcu **slot;714 715 radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {716 if (uapi_key_is_object(iter.index)) {717 struct uverbs_api_object *object_elm =718 rcu_dereference_protected(*slot, true);719 720 /*721 * Some type_attrs are in the driver module. We don't722 * bother to keep track of which since there should be723 * no use of this after disassociate.724 */725 object_elm->type_attrs = NULL;726 } else if (uapi_key_is_attr(iter.index)) {727 struct uverbs_api_attr *elm =728 rcu_dereference_protected(*slot, true);729 730 if (elm->spec.type == UVERBS_ATTR_TYPE_ENUM_IN)731 elm->spec.u2.enum_def.ids = NULL;732 }733 }734}735