brintos / linux-shallow public Read only

1/*2 * Copyright (c) 2005 Topspin Communications.  All rights reserved.3 * Copyright (c) 2005, 2006, 2007 Cisco Systems.  All rights reserved.4 * Copyright (c) 2005 PathScale, Inc.  All rights reserved.5 * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.6 *7 * This software is available to you under a choice of one of two8 * licenses.  You may choose to be licensed under the terms of the GNU9 * General Public License (GPL) Version 2, available from the file10 * COPYING in the main directory of this source tree, or the11 * OpenIB.org BSD license below:12 *13 *     Redistribution and use in source and binary forms, with or14 *     without modification, are permitted provided that the following15 *     conditions are met:16 *17 *      - Redistributions of source code must retain the above18 *        copyright notice, this list of conditions and the following19 *        disclaimer.20 *21 *      - Redistributions in binary form must reproduce the above22 *        copyright notice, this list of conditions and the following23 *        disclaimer in the documentation and/or other materials24 *        provided with the distribution.25 *26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE33 * SOFTWARE.34 */35 36#include <linux/file.h>37#include <linux/fs.h>38#include <linux/slab.h>39#include <linux/sched.h>40 41#include <linux/uaccess.h>42 43#include <rdma/uverbs_types.h>44#include <rdma/uverbs_std_types.h>45#include "rdma_core.h"46 47#include "uverbs.h"48#include "core_priv.h"49 50/*51 * Copy a response to userspace. If the provided 'resp' is larger than the52 * user buffer it is silently truncated. If the user provided a larger buffer53 * then the trailing portion is zero filled.54 *55 * These semantics are intended to support future extension of the output56 * structures.57 */58static int uverbs_response(struct uverbs_attr_bundle *attrs, const void *resp,59			   size_t resp_len)60{61	int ret;62 63	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))64		return uverbs_copy_to_struct_or_zero(65			attrs, UVERBS_ATTR_CORE_OUT, resp, resp_len);66 67	if (copy_to_user(attrs->ucore.outbuf, resp,68			 min(attrs->ucore.outlen, resp_len)))69		return -EFAULT;70 71	if (resp_len < attrs->ucore.outlen) {72		/*73		 * Zero fill any extra memory that user74		 * space might have provided.75		 */76		ret = clear_user(attrs->ucore.outbuf + resp_len,77				 attrs->ucore.outlen - resp_len);78		if (ret)79			return -EFAULT;80	}81 82	return 0;83}84 85/*86 * Copy a request from userspace. If the provided 'req' is larger than the87 * user buffer then the user buffer is zero extended into the 'req'. If 'req'88 * is smaller than the user buffer then the uncopied bytes in the user buffer89 * must be zero.90 */91static int uverbs_request(struct uverbs_attr_bundle *attrs, void *req,92			  size_t req_len)93{94	if (copy_from_user(req, attrs->ucore.inbuf,95			   min(attrs->ucore.inlen, req_len)))96		return -EFAULT;97 98	if (attrs->ucore.inlen < req_len) {99		memset(req + attrs->ucore.inlen, 0,100		       req_len - attrs->ucore.inlen);101	} else if (attrs->ucore.inlen > req_len) {102		if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len,103					  attrs->ucore.inlen - req_len))104			return -EOPNOTSUPP;105	}106	return 0;107}108 109/*110 * Generate the value for the 'response_length' protocol used by write_ex.111 * This is the number of bytes the kernel actually wrote. Userspace can use112 * this to detect what structure members in the response the kernel113 * understood.114 */115static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs,116				  size_t resp_len)117{118	return min_t(size_t, attrs->ucore.outlen, resp_len);119}120 121/*122 * The iterator version of the request interface is for handlers that need to123 * step over a flex array at the end of a command header.124 */125struct uverbs_req_iter {126	const void __user *cur;127	const void __user *end;128};129 130static int uverbs_request_start(struct uverbs_attr_bundle *attrs,131				struct uverbs_req_iter *iter,132				void *req,133				size_t req_len)134{135	if (attrs->ucore.inlen < req_len)136		return -ENOSPC;137 138	if (copy_from_user(req, attrs->ucore.inbuf, req_len))139		return -EFAULT;140 141	iter->cur = attrs->ucore.inbuf + req_len;142	iter->end = attrs->ucore.inbuf + attrs->ucore.inlen;143	return 0;144}145 146static int uverbs_request_next(struct uverbs_req_iter *iter, void *val,147			       size_t len)148{149	if (iter->cur + len > iter->end)150		return -ENOSPC;151 152	if (copy_from_user(val, iter->cur, len))153		return -EFAULT;154 155	iter->cur += len;156	return 0;157}158 159static const void __user *uverbs_request_next_ptr(struct uverbs_req_iter *iter,160						  size_t len)161{162	const void __user *res = iter->cur;163 164	if (iter->cur + len > iter->end)165		return (void __force __user *)ERR_PTR(-ENOSPC);166	iter->cur += len;167	return res;168}169 170static int uverbs_request_finish(struct uverbs_req_iter *iter)171{172	if (!ib_is_buffer_cleared(iter->cur, iter->end - iter->cur))173		return -EOPNOTSUPP;174	return 0;175}176 177/*178 * When calling a destroy function during an error unwind we need to pass in179 * the udata that is sanitized of all user arguments. Ie from the driver180 * perspective it looks like no udata was passed.181 */182struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)183{184	attrs->driver_udata = (struct ib_udata){};185	return &attrs->driver_udata;186}187 188static struct ib_uverbs_completion_event_file *189_ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)190{191	struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL,192					       fd, attrs);193 194	if (IS_ERR(uobj))195		return (void *)uobj;196 197	uverbs_uobject_get(uobj);198	uobj_put_read(uobj);199 200	return container_of(uobj, struct ib_uverbs_completion_event_file,201			    uobj);202}203#define ib_uverbs_lookup_comp_file(_fd, _ufile)                                \204	_ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile)205 206int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs)207{208	struct ib_uverbs_file *ufile = attrs->ufile;209	struct ib_ucontext *ucontext;210	struct ib_device *ib_dev;211 212	ib_dev = srcu_dereference(ufile->device->ib_dev,213				  &ufile->device->disassociate_srcu);214	if (!ib_dev)215		return -EIO;216 217	ucontext = rdma_zalloc_drv_obj(ib_dev, ib_ucontext);218	if (!ucontext)219		return -ENOMEM;220 221	ucontext->device = ib_dev;222	ucontext->ufile = ufile;223	xa_init_flags(&ucontext->mmap_xa, XA_FLAGS_ALLOC);224 225	rdma_restrack_new(&ucontext->res, RDMA_RESTRACK_CTX);226	rdma_restrack_set_name(&ucontext->res, NULL);227	attrs->context = ucontext;228	return 0;229}230 231int ib_init_ucontext(struct uverbs_attr_bundle *attrs)232{233	struct ib_ucontext *ucontext = attrs->context;234	struct ib_uverbs_file *file = attrs->ufile;235	int ret;236 237	if (!down_read_trylock(&file->hw_destroy_rwsem))238		return -EIO;239	mutex_lock(&file->ucontext_lock);240	if (file->ucontext) {241		ret = -EINVAL;242		goto err;243	}244 245	ret = ib_rdmacg_try_charge(&ucontext->cg_obj, ucontext->device,246				   RDMACG_RESOURCE_HCA_HANDLE);247	if (ret)248		goto err;249 250	ret = ucontext->device->ops.alloc_ucontext(ucontext,251						   &attrs->driver_udata);252	if (ret)253		goto err_uncharge;254 255	rdma_restrack_add(&ucontext->res);256 257	/*258	 * Make sure that ib_uverbs_get_ucontext() sees the pointer update259	 * only after all writes to setup the ucontext have completed260	 */261	smp_store_release(&file->ucontext, ucontext);262 263	mutex_unlock(&file->ucontext_lock);264	up_read(&file->hw_destroy_rwsem);265	return 0;266 267err_uncharge:268	ib_rdmacg_uncharge(&ucontext->cg_obj, ucontext->device,269			   RDMACG_RESOURCE_HCA_HANDLE);270err:271	mutex_unlock(&file->ucontext_lock);272	up_read(&file->hw_destroy_rwsem);273	return ret;274}275 276static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)277{278	struct ib_uverbs_get_context_resp resp;279	struct ib_uverbs_get_context cmd;280	struct ib_device *ib_dev;281	struct ib_uobject *uobj;282	int ret;283 284	ret = uverbs_request(attrs, &cmd, sizeof(cmd));285	if (ret)286		return ret;287 288	ret = ib_alloc_ucontext(attrs);289	if (ret)290		return ret;291 292	uobj = uobj_alloc(UVERBS_OBJECT_ASYNC_EVENT, attrs, &ib_dev);293	if (IS_ERR(uobj)) {294		ret = PTR_ERR(uobj);295		goto err_ucontext;296	}297 298	resp = (struct ib_uverbs_get_context_resp){299		.num_comp_vectors = attrs->ufile->device->num_comp_vectors,300		.async_fd = uobj->id,301	};302	ret = uverbs_response(attrs, &resp, sizeof(resp));303	if (ret)304		goto err_uobj;305 306	ret = ib_init_ucontext(attrs);307	if (ret)308		goto err_uobj;309 310	ib_uverbs_init_async_event_file(311		container_of(uobj, struct ib_uverbs_async_event_file, uobj));312	rdma_alloc_commit_uobject(uobj, attrs);313	return 0;314 315err_uobj:316	rdma_alloc_abort_uobject(uobj, attrs, false);317err_ucontext:318	rdma_restrack_put(&attrs->context->res);319	kfree(attrs->context);320	attrs->context = NULL;321	return ret;322}323 324static void copy_query_dev_fields(struct ib_ucontext *ucontext,325				  struct ib_uverbs_query_device_resp *resp,326				  struct ib_device_attr *attr)327{328	struct ib_device *ib_dev = ucontext->device;329 330	resp->fw_ver		= attr->fw_ver;331	resp->node_guid		= ib_dev->node_guid;332	resp->sys_image_guid	= attr->sys_image_guid;333	resp->max_mr_size	= attr->max_mr_size;334	resp->page_size_cap	= attr->page_size_cap;335	resp->vendor_id		= attr->vendor_id;336	resp->vendor_part_id	= attr->vendor_part_id;337	resp->hw_ver		= attr->hw_ver;338	resp->max_qp		= attr->max_qp;339	resp->max_qp_wr		= attr->max_qp_wr;340	resp->device_cap_flags  = lower_32_bits(attr->device_cap_flags);341	resp->max_sge		= min(attr->max_send_sge, attr->max_recv_sge);342	resp->max_sge_rd	= attr->max_sge_rd;343	resp->max_cq		= attr->max_cq;344	resp->max_cqe		= attr->max_cqe;345	resp->max_mr		= attr->max_mr;346	resp->max_pd		= attr->max_pd;347	resp->max_qp_rd_atom	= attr->max_qp_rd_atom;348	resp->max_ee_rd_atom	= attr->max_ee_rd_atom;349	resp->max_res_rd_atom	= attr->max_res_rd_atom;350	resp->max_qp_init_rd_atom	= attr->max_qp_init_rd_atom;351	resp->max_ee_init_rd_atom	= attr->max_ee_init_rd_atom;352	resp->atomic_cap		= attr->atomic_cap;353	resp->max_ee			= attr->max_ee;354	resp->max_rdd			= attr->max_rdd;355	resp->max_mw			= attr->max_mw;356	resp->max_raw_ipv6_qp		= attr->max_raw_ipv6_qp;357	resp->max_raw_ethy_qp		= attr->max_raw_ethy_qp;358	resp->max_mcast_grp		= attr->max_mcast_grp;359	resp->max_mcast_qp_attach	= attr->max_mcast_qp_attach;360	resp->max_total_mcast_qp_attach	= attr->max_total_mcast_qp_attach;361	resp->max_ah			= attr->max_ah;362	resp->max_srq			= attr->max_srq;363	resp->max_srq_wr		= attr->max_srq_wr;364	resp->max_srq_sge		= attr->max_srq_sge;365	resp->max_pkeys			= attr->max_pkeys;366	resp->local_ca_ack_delay	= attr->local_ca_ack_delay;367	resp->phys_port_cnt = min_t(u32, ib_dev->phys_port_cnt, U8_MAX);368}369 370static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs)371{372	struct ib_uverbs_query_device      cmd;373	struct ib_uverbs_query_device_resp resp;374	struct ib_ucontext *ucontext;375	int ret;376 377	ucontext = ib_uverbs_get_ucontext(attrs);378	if (IS_ERR(ucontext))379		return PTR_ERR(ucontext);380 381	ret = uverbs_request(attrs, &cmd, sizeof(cmd));382	if (ret)383		return ret;384 385	memset(&resp, 0, sizeof resp);386	copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs);387 388	return uverbs_response(attrs, &resp, sizeof(resp));389}390 391static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs)392{393	struct ib_uverbs_query_port      cmd;394	struct ib_uverbs_query_port_resp resp;395	struct ib_port_attr              attr;396	int                              ret;397	struct ib_ucontext *ucontext;398	struct ib_device *ib_dev;399 400	ucontext = ib_uverbs_get_ucontext(attrs);401	if (IS_ERR(ucontext))402		return PTR_ERR(ucontext);403	ib_dev = ucontext->device;404 405	ret = uverbs_request(attrs, &cmd, sizeof(cmd));406	if (ret)407		return ret;408 409	ret = ib_query_port(ib_dev, cmd.port_num, &attr);410	if (ret)411		return ret;412 413	memset(&resp, 0, sizeof resp);414	copy_port_attr_to_resp(&attr, &resp, ib_dev, cmd.port_num);415 416	return uverbs_response(attrs, &resp, sizeof(resp));417}418 419static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)420{421	struct ib_uverbs_alloc_pd_resp resp = {};422	struct ib_uverbs_alloc_pd      cmd;423	struct ib_uobject             *uobj;424	struct ib_pd                  *pd;425	int                            ret;426	struct ib_device *ib_dev;427 428	ret = uverbs_request(attrs, &cmd, sizeof(cmd));429	if (ret)430		return ret;431 432	uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev);433	if (IS_ERR(uobj))434		return PTR_ERR(uobj);435 436	pd = rdma_zalloc_drv_obj(ib_dev, ib_pd);437	if (!pd) {438		ret = -ENOMEM;439		goto err;440	}441 442	pd->device  = ib_dev;443	pd->uobject = uobj;444	atomic_set(&pd->usecnt, 0);445 446	rdma_restrack_new(&pd->res, RDMA_RESTRACK_PD);447	rdma_restrack_set_name(&pd->res, NULL);448 449	ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);450	if (ret)451		goto err_alloc;452	rdma_restrack_add(&pd->res);453 454	uobj->object = pd;455	uobj_finalize_uobj_create(uobj, attrs);456 457	resp.pd_handle = uobj->id;458	return uverbs_response(attrs, &resp, sizeof(resp));459 460err_alloc:461	rdma_restrack_put(&pd->res);462	kfree(pd);463err:464	uobj_alloc_abort(uobj, attrs);465	return ret;466}467 468static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs)469{470	struct ib_uverbs_dealloc_pd cmd;471	int ret;472 473	ret = uverbs_request(attrs, &cmd, sizeof(cmd));474	if (ret)475		return ret;476 477	return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs);478}479 480struct xrcd_table_entry {481	struct rb_node  node;482	struct ib_xrcd *xrcd;483	struct inode   *inode;484};485 486static int xrcd_table_insert(struct ib_uverbs_device *dev,487			    struct inode *inode,488			    struct ib_xrcd *xrcd)489{490	struct xrcd_table_entry *entry, *scan;491	struct rb_node **p = &dev->xrcd_tree.rb_node;492	struct rb_node *parent = NULL;493 494	entry = kmalloc(sizeof *entry, GFP_KERNEL);495	if (!entry)496		return -ENOMEM;497 498	entry->xrcd  = xrcd;499	entry->inode = inode;500 501	while (*p) {502		parent = *p;503		scan = rb_entry(parent, struct xrcd_table_entry, node);504 505		if (inode < scan->inode) {506			p = &(*p)->rb_left;507		} else if (inode > scan->inode) {508			p = &(*p)->rb_right;509		} else {510			kfree(entry);511			return -EEXIST;512		}513	}514 515	rb_link_node(&entry->node, parent, p);516	rb_insert_color(&entry->node, &dev->xrcd_tree);517	igrab(inode);518	return 0;519}520 521static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev,522						  struct inode *inode)523{524	struct xrcd_table_entry *entry;525	struct rb_node *p = dev->xrcd_tree.rb_node;526 527	while (p) {528		entry = rb_entry(p, struct xrcd_table_entry, node);529 530		if (inode < entry->inode)531			p = p->rb_left;532		else if (inode > entry->inode)533			p = p->rb_right;534		else535			return entry;536	}537 538	return NULL;539}540 541static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode)542{543	struct xrcd_table_entry *entry;544 545	entry = xrcd_table_search(dev, inode);546	if (!entry)547		return NULL;548 549	return entry->xrcd;550}551 552static void xrcd_table_delete(struct ib_uverbs_device *dev,553			      struct inode *inode)554{555	struct xrcd_table_entry *entry;556 557	entry = xrcd_table_search(dev, inode);558	if (entry) {559		iput(inode);560		rb_erase(&entry->node, &dev->xrcd_tree);561		kfree(entry);562	}563}564 565static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)566{567	struct ib_uverbs_device *ibudev = attrs->ufile->device;568	struct ib_uverbs_open_xrcd_resp	resp = {};569	struct ib_uverbs_open_xrcd	cmd;570	struct ib_uxrcd_object         *obj;571	struct ib_xrcd                 *xrcd = NULL;572	struct inode                   *inode = NULL;573	int				new_xrcd = 0;574	struct ib_device *ib_dev;575	struct fd f = EMPTY_FD;576	int ret;577 578	ret = uverbs_request(attrs, &cmd, sizeof(cmd));579	if (ret)580		return ret;581 582	mutex_lock(&ibudev->xrcd_tree_mutex);583 584	if (cmd.fd != -1) {585		/* search for file descriptor */586		f = fdget(cmd.fd);587		if (!fd_file(f)) {588			ret = -EBADF;589			goto err_tree_mutex_unlock;590		}591 592		inode = file_inode(fd_file(f));593		xrcd = find_xrcd(ibudev, inode);594		if (!xrcd && !(cmd.oflags & O_CREAT)) {595			/* no file descriptor. Need CREATE flag */596			ret = -EAGAIN;597			goto err_tree_mutex_unlock;598		}599 600		if (xrcd && cmd.oflags & O_EXCL) {601			ret = -EINVAL;602			goto err_tree_mutex_unlock;603		}604	}605 606	obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs,607						   &ib_dev);608	if (IS_ERR(obj)) {609		ret = PTR_ERR(obj);610		goto err_tree_mutex_unlock;611	}612 613	if (!xrcd) {614		xrcd = ib_alloc_xrcd_user(ib_dev, inode, &attrs->driver_udata);615		if (IS_ERR(xrcd)) {616			ret = PTR_ERR(xrcd);617			goto err;618		}619		new_xrcd = 1;620	}621 622	atomic_set(&obj->refcnt, 0);623	obj->uobject.object = xrcd;624 625	if (inode) {626		if (new_xrcd) {627			/* create new inode/xrcd table entry */628			ret = xrcd_table_insert(ibudev, inode, xrcd);629			if (ret)630				goto err_dealloc_xrcd;631		}632		atomic_inc(&xrcd->usecnt);633	}634 635	if (fd_file(f))636		fdput(f);637 638	mutex_unlock(&ibudev->xrcd_tree_mutex);639	uobj_finalize_uobj_create(&obj->uobject, attrs);640 641	resp.xrcd_handle = obj->uobject.id;642	return uverbs_response(attrs, &resp, sizeof(resp));643 644err_dealloc_xrcd:645	ib_dealloc_xrcd_user(xrcd, uverbs_get_cleared_udata(attrs));646 647err:648	uobj_alloc_abort(&obj->uobject, attrs);649 650err_tree_mutex_unlock:651	if (fd_file(f))652		fdput(f);653 654	mutex_unlock(&ibudev->xrcd_tree_mutex);655 656	return ret;657}658 659static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs)660{661	struct ib_uverbs_close_xrcd cmd;662	int ret;663 664	ret = uverbs_request(attrs, &cmd, sizeof(cmd));665	if (ret)666		return ret;667 668	return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs);669}670 671int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd,672			   enum rdma_remove_reason why,673			   struct uverbs_attr_bundle *attrs)674{675	struct inode *inode;676	int ret;677	struct ib_uverbs_device *dev = attrs->ufile->device;678 679	inode = xrcd->inode;680	if (inode && !atomic_dec_and_test(&xrcd->usecnt))681		return 0;682 683	ret = ib_dealloc_xrcd_user(xrcd, &attrs->driver_udata);684	if (ret) {685		atomic_inc(&xrcd->usecnt);686		return ret;687	}688 689	if (inode)690		xrcd_table_delete(dev, inode);691 692	return 0;693}694 695static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)696{697	struct ib_uverbs_reg_mr_resp resp = {};698	struct ib_uverbs_reg_mr      cmd;699	struct ib_uobject           *uobj;700	struct ib_pd                *pd;701	struct ib_mr                *mr;702	int                          ret;703	struct ib_device *ib_dev;704 705	ret = uverbs_request(attrs, &cmd, sizeof(cmd));706	if (ret)707		return ret;708 709	if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))710		return -EINVAL;711 712	uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);713	if (IS_ERR(uobj))714		return PTR_ERR(uobj);715 716	ret = ib_check_mr_access(ib_dev, cmd.access_flags);717	if (ret)718		goto err_free;719 720	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);721	if (!pd) {722		ret = -EINVAL;723		goto err_free;724	}725 726	mr = pd->device->ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,727					 cmd.access_flags,728					 &attrs->driver_udata);729	if (IS_ERR(mr)) {730		ret = PTR_ERR(mr);731		goto err_put;732	}733 734	mr->device  = pd->device;735	mr->pd      = pd;736	mr->type    = IB_MR_TYPE_USER;737	mr->dm	    = NULL;738	mr->sig_attrs = NULL;739	mr->uobject = uobj;740	atomic_inc(&pd->usecnt);741	mr->iova = cmd.hca_va;742	mr->length = cmd.length;743 744	rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);745	rdma_restrack_set_name(&mr->res, NULL);746	rdma_restrack_add(&mr->res);747 748	uobj->object = mr;749	uobj_put_obj_read(pd);750	uobj_finalize_uobj_create(uobj, attrs);751 752	resp.lkey = mr->lkey;753	resp.rkey = mr->rkey;754	resp.mr_handle = uobj->id;755	return uverbs_response(attrs, &resp, sizeof(resp));756 757err_put:758	uobj_put_obj_read(pd);759err_free:760	uobj_alloc_abort(uobj, attrs);761	return ret;762}763 764static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)765{766	struct ib_uverbs_rereg_mr      cmd;767	struct ib_uverbs_rereg_mr_resp resp;768	struct ib_mr                *mr;769	int                          ret;770	struct ib_uobject	    *uobj;771	struct ib_uobject *new_uobj;772	struct ib_device *ib_dev;773	struct ib_pd *orig_pd;774	struct ib_pd *new_pd;775	struct ib_mr *new_mr;776 777	ret = uverbs_request(attrs, &cmd, sizeof(cmd));778	if (ret)779		return ret;780 781	if (!cmd.flags)782		return -EINVAL;783 784	if (cmd.flags & ~IB_MR_REREG_SUPPORTED)785		return -EOPNOTSUPP;786 787	if ((cmd.flags & IB_MR_REREG_TRANS) &&788	    (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))789		return -EINVAL;790 791	uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);792	if (IS_ERR(uobj))793		return PTR_ERR(uobj);794 795	mr = uobj->object;796 797	if (mr->dm) {798		ret = -EINVAL;799		goto put_uobjs;800	}801 802	if (cmd.flags & IB_MR_REREG_ACCESS) {803		ret = ib_check_mr_access(mr->device, cmd.access_flags);804		if (ret)805			goto put_uobjs;806	}807 808	orig_pd = mr->pd;809	if (cmd.flags & IB_MR_REREG_PD) {810		new_pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle,811					   attrs);812		if (!new_pd) {813			ret = -EINVAL;814			goto put_uobjs;815		}816	} else {817		new_pd = mr->pd;818	}819 820	/*821	 * The driver might create a new HW object as part of the rereg, we need822	 * to have a uobject ready to hold it.823	 */824	new_uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);825	if (IS_ERR(new_uobj)) {826		ret = PTR_ERR(new_uobj);827		goto put_uobj_pd;828	}829 830	new_mr = ib_dev->ops.rereg_user_mr(mr, cmd.flags, cmd.start, cmd.length,831					   cmd.hca_va, cmd.access_flags, new_pd,832					   &attrs->driver_udata);833	if (IS_ERR(new_mr)) {834		ret = PTR_ERR(new_mr);835		goto put_new_uobj;836	}837	if (new_mr) {838		new_mr->device = new_pd->device;839		new_mr->pd = new_pd;840		new_mr->type = IB_MR_TYPE_USER;841		new_mr->uobject = uobj;842		atomic_inc(&new_pd->usecnt);843		new_uobj->object = new_mr;844 845		rdma_restrack_new(&new_mr->res, RDMA_RESTRACK_MR);846		rdma_restrack_set_name(&new_mr->res, NULL);847		rdma_restrack_add(&new_mr->res);848 849		/*850		 * The new uobj for the new HW object is put into the same spot851		 * in the IDR and the old uobj & HW object is deleted.852		 */853		rdma_assign_uobject(uobj, new_uobj, attrs);854		rdma_alloc_commit_uobject(new_uobj, attrs);855		uobj_put_destroy(uobj);856		new_uobj = NULL;857		uobj = NULL;858		mr = new_mr;859	} else {860		if (cmd.flags & IB_MR_REREG_PD) {861			atomic_dec(&orig_pd->usecnt);862			mr->pd = new_pd;863			atomic_inc(&new_pd->usecnt);864		}865		if (cmd.flags & IB_MR_REREG_TRANS) {866			mr->iova = cmd.hca_va;867			mr->length = cmd.length;868		}869	}870 871	memset(&resp, 0, sizeof(resp));872	resp.lkey      = mr->lkey;873	resp.rkey      = mr->rkey;874 875	ret = uverbs_response(attrs, &resp, sizeof(resp));876 877put_new_uobj:878	if (new_uobj)879		uobj_alloc_abort(new_uobj, attrs);880put_uobj_pd:881	if (cmd.flags & IB_MR_REREG_PD)882		uobj_put_obj_read(new_pd);883 884put_uobjs:885	if (uobj)886		uobj_put_write(uobj);887 888	return ret;889}890 891static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs)892{893	struct ib_uverbs_dereg_mr cmd;894	int ret;895 896	ret = uverbs_request(attrs, &cmd, sizeof(cmd));897	if (ret)898		return ret;899 900	return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);901}902 903static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)904{905	struct ib_uverbs_alloc_mw      cmd;906	struct ib_uverbs_alloc_mw_resp resp = {};907	struct ib_uobject             *uobj;908	struct ib_pd                  *pd;909	struct ib_mw                  *mw;910	int                            ret;911	struct ib_device *ib_dev;912 913	ret = uverbs_request(attrs, &cmd, sizeof(cmd));914	if (ret)915		return ret;916 917	uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev);918	if (IS_ERR(uobj))919		return PTR_ERR(uobj);920 921	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);922	if (!pd) {923		ret = -EINVAL;924		goto err_free;925	}926 927	if (cmd.mw_type != IB_MW_TYPE_1 && cmd.mw_type != IB_MW_TYPE_2) {928		ret = -EINVAL;929		goto err_put;930	}931 932	mw = rdma_zalloc_drv_obj(ib_dev, ib_mw);933	if (!mw) {934		ret = -ENOMEM;935		goto err_put;936	}937 938	mw->device = ib_dev;939	mw->pd = pd;940	mw->uobject = uobj;941	mw->type = cmd.mw_type;942 943	ret = pd->device->ops.alloc_mw(mw, &attrs->driver_udata);944	if (ret)945		goto err_alloc;946 947	atomic_inc(&pd->usecnt);948 949	uobj->object = mw;950	uobj_put_obj_read(pd);951	uobj_finalize_uobj_create(uobj, attrs);952 953	resp.rkey = mw->rkey;954	resp.mw_handle = uobj->id;955	return uverbs_response(attrs, &resp, sizeof(resp));956 957err_alloc:958	kfree(mw);959err_put:960	uobj_put_obj_read(pd);961err_free:962	uobj_alloc_abort(uobj, attrs);963	return ret;964}965 966static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs)967{968	struct ib_uverbs_dealloc_mw cmd;969	int ret;970 971	ret = uverbs_request(attrs, &cmd, sizeof(cmd));972	if (ret)973		return ret;974 975	return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs);976}977 978static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)979{980	struct ib_uverbs_create_comp_channel	   cmd;981	struct ib_uverbs_create_comp_channel_resp  resp;982	struct ib_uobject			  *uobj;983	struct ib_uverbs_completion_event_file	  *ev_file;984	struct ib_device *ib_dev;985	int ret;986 987	ret = uverbs_request(attrs, &cmd, sizeof(cmd));988	if (ret)989		return ret;990 991	uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev);992	if (IS_ERR(uobj))993		return PTR_ERR(uobj);994 995	ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,996			       uobj);997	ib_uverbs_init_event_queue(&ev_file->ev_queue);998	uobj_finalize_uobj_create(uobj, attrs);999 1000	resp.fd = uobj->id;1001	return uverbs_response(attrs, &resp, sizeof(resp));1002}1003 1004static int create_cq(struct uverbs_attr_bundle *attrs,1005		     struct ib_uverbs_ex_create_cq *cmd)1006{1007	struct ib_ucq_object           *obj;1008	struct ib_uverbs_completion_event_file    *ev_file = NULL;1009	struct ib_cq                   *cq;1010	int                             ret;1011	struct ib_uverbs_ex_create_cq_resp resp = {};1012	struct ib_cq_init_attr attr = {};1013	struct ib_device *ib_dev;1014 1015	if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)1016		return -EINVAL;1017 1018	obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,1019						 &ib_dev);1020	if (IS_ERR(obj))1021		return PTR_ERR(obj);1022 1023	if (cmd->comp_channel >= 0) {1024		ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);1025		if (IS_ERR(ev_file)) {1026			ret = PTR_ERR(ev_file);1027			goto err;1028		}1029	}1030 1031	obj->uevent.uobject.user_handle = cmd->user_handle;1032	INIT_LIST_HEAD(&obj->comp_list);1033	INIT_LIST_HEAD(&obj->uevent.event_list);1034 1035	attr.cqe = cmd->cqe;1036	attr.comp_vector = cmd->comp_vector;1037	attr.flags = cmd->flags;1038 1039	cq = rdma_zalloc_drv_obj(ib_dev, ib_cq);1040	if (!cq) {1041		ret = -ENOMEM;1042		goto err_file;1043	}1044	cq->device        = ib_dev;1045	cq->uobject       = obj;1046	cq->comp_handler  = ib_uverbs_comp_handler;1047	cq->event_handler = ib_uverbs_cq_event_handler;1048	cq->cq_context    = ev_file ? &ev_file->ev_queue : NULL;1049	atomic_set(&cq->usecnt, 0);1050 1051	rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);1052	rdma_restrack_set_name(&cq->res, NULL);1053 1054	ret = ib_dev->ops.create_cq(cq, &attr, attrs);1055	if (ret)1056		goto err_free;1057	rdma_restrack_add(&cq->res);1058 1059	obj->uevent.uobject.object = cq;1060	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);1061	if (obj->uevent.event_file)1062		uverbs_uobject_get(&obj->uevent.event_file->uobj);1063	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);1064 1065	resp.base.cq_handle = obj->uevent.uobject.id;1066	resp.base.cqe = cq->cqe;1067	resp.response_length = uverbs_response_length(attrs, sizeof(resp));1068	return uverbs_response(attrs, &resp, sizeof(resp));1069 1070err_free:1071	rdma_restrack_put(&cq->res);1072	kfree(cq);1073err_file:1074	if (ev_file)1075		ib_uverbs_release_ucq(ev_file, obj);1076err:1077	uobj_alloc_abort(&obj->uevent.uobject, attrs);1078	return ret;1079}1080 1081static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)1082{1083	struct ib_uverbs_create_cq      cmd;1084	struct ib_uverbs_ex_create_cq	cmd_ex;1085	int ret;1086 1087	ret = uverbs_request(attrs, &cmd, sizeof(cmd));1088	if (ret)1089		return ret;1090 1091	memset(&cmd_ex, 0, sizeof(cmd_ex));1092	cmd_ex.user_handle = cmd.user_handle;1093	cmd_ex.cqe = cmd.cqe;1094	cmd_ex.comp_vector = cmd.comp_vector;1095	cmd_ex.comp_channel = cmd.comp_channel;1096 1097	return create_cq(attrs, &cmd_ex);1098}1099 1100static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)1101{1102	struct ib_uverbs_ex_create_cq  cmd;1103	int ret;1104 1105	ret = uverbs_request(attrs, &cmd, sizeof(cmd));1106	if (ret)1107		return ret;1108 1109	if (cmd.comp_mask)1110		return -EINVAL;1111 1112	if (cmd.reserved)1113		return -EINVAL;1114 1115	return create_cq(attrs, &cmd);1116}1117 1118static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)1119{1120	struct ib_uverbs_resize_cq	cmd;1121	struct ib_uverbs_resize_cq_resp	resp = {};1122	struct ib_cq			*cq;1123	int ret;1124 1125	ret = uverbs_request(attrs, &cmd, sizeof(cmd));1126	if (ret)1127		return ret;1128 1129	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);1130	if (!cq)1131		return -EINVAL;1132 1133	ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata);1134	if (ret)1135		goto out;1136 1137	resp.cqe = cq->cqe;1138 1139	ret = uverbs_response(attrs, &resp, sizeof(resp));1140out:1141	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,1142				UVERBS_LOOKUP_READ);1143 1144	return ret;1145}1146 1147static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest,1148			   struct ib_wc *wc)1149{1150	struct ib_uverbs_wc tmp;1151 1152	tmp.wr_id		= wc->wr_id;1153	tmp.status		= wc->status;1154	tmp.opcode		= wc->opcode;1155	tmp.vendor_err		= wc->vendor_err;1156	tmp.byte_len		= wc->byte_len;1157	tmp.ex.imm_data		= wc->ex.imm_data;1158	tmp.qp_num		= wc->qp->qp_num;1159	tmp.src_qp		= wc->src_qp;1160	tmp.wc_flags		= wc->wc_flags;1161	tmp.pkey_index		= wc->pkey_index;1162	if (rdma_cap_opa_ah(ib_dev, wc->port_num))1163		tmp.slid	= OPA_TO_IB_UCAST_LID(wc->slid);1164	else1165		tmp.slid	= ib_lid_cpu16(wc->slid);1166	tmp.sl			= wc->sl;1167	tmp.dlid_path_bits	= wc->dlid_path_bits;1168	tmp.port_num		= wc->port_num;1169	tmp.reserved		= 0;1170 1171	if (copy_to_user(dest, &tmp, sizeof tmp))1172		return -EFAULT;1173 1174	return 0;1175}1176 1177static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)1178{1179	struct ib_uverbs_poll_cq       cmd;1180	struct ib_uverbs_poll_cq_resp  resp;1181	u8 __user                     *header_ptr;1182	u8 __user                     *data_ptr;1183	struct ib_cq                  *cq;1184	struct ib_wc                   wc;1185	int                            ret;1186 1187	ret = uverbs_request(attrs, &cmd, sizeof(cmd));1188	if (ret)1189		return ret;1190 1191	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);1192	if (!cq)1193		return -EINVAL;1194 1195	/* we copy a struct ib_uverbs_poll_cq_resp to user space */1196	header_ptr = attrs->ucore.outbuf;1197	data_ptr = header_ptr + sizeof resp;1198 1199	memset(&resp, 0, sizeof resp);1200	while (resp.count < cmd.ne) {1201		ret = ib_poll_cq(cq, 1, &wc);1202		if (ret < 0)1203			goto out_put;1204		if (!ret)1205			break;1206 1207		ret = copy_wc_to_user(cq->device, data_ptr, &wc);1208		if (ret)1209			goto out_put;1210 1211		data_ptr += sizeof(struct ib_uverbs_wc);1212		++resp.count;1213	}1214 1215	if (copy_to_user(header_ptr, &resp, sizeof resp)) {1216		ret = -EFAULT;1217		goto out_put;1218	}1219	ret = 0;1220 1221	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))1222		ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT);1223 1224out_put:1225	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,1226				UVERBS_LOOKUP_READ);1227	return ret;1228}1229 1230static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs)1231{1232	struct ib_uverbs_req_notify_cq cmd;1233	struct ib_cq                  *cq;1234	int ret;1235 1236	ret = uverbs_request(attrs, &cmd, sizeof(cmd));1237	if (ret)1238		return ret;1239 1240	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);1241	if (!cq)1242		return -EINVAL;1243 1244	ib_req_notify_cq(cq, cmd.solicited_only ?1245			 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);1246 1247	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,1248				UVERBS_LOOKUP_READ);1249	return 0;1250}1251 1252static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs)1253{1254	struct ib_uverbs_destroy_cq      cmd;1255	struct ib_uverbs_destroy_cq_resp resp;1256	struct ib_uobject		*uobj;1257	struct ib_ucq_object        	*obj;1258	int ret;1259 1260	ret = uverbs_request(attrs, &cmd, sizeof(cmd));1261	if (ret)1262		return ret;1263 1264	uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);1265	if (IS_ERR(uobj))1266		return PTR_ERR(uobj);1267 1268	obj = container_of(uobj, struct ib_ucq_object, uevent.uobject);1269	memset(&resp, 0, sizeof(resp));1270	resp.comp_events_reported  = obj->comp_events_reported;1271	resp.async_events_reported = obj->uevent.events_reported;1272 1273	uobj_put_destroy(uobj);1274 1275	return uverbs_response(attrs, &resp, sizeof(resp));1276}1277 1278static int create_qp(struct uverbs_attr_bundle *attrs,1279		     struct ib_uverbs_ex_create_qp *cmd)1280{1281	struct ib_uqp_object		*obj;1282	struct ib_device		*device;1283	struct ib_pd			*pd = NULL;1284	struct ib_xrcd			*xrcd = NULL;1285	struct ib_uobject		*xrcd_uobj = ERR_PTR(-ENOENT);1286	struct ib_cq			*scq = NULL, *rcq = NULL;1287	struct ib_srq			*srq = NULL;1288	struct ib_qp			*qp;1289	struct ib_qp_init_attr		attr = {};1290	struct ib_uverbs_ex_create_qp_resp resp = {};1291	int				ret;1292	struct ib_rwq_ind_table *ind_tbl = NULL;1293	bool has_sq = true;1294	struct ib_device *ib_dev;1295 1296	switch (cmd->qp_type) {1297	case IB_QPT_RAW_PACKET:1298		if (!capable(CAP_NET_RAW))1299			return -EPERM;1300		break;1301	case IB_QPT_RC:1302	case IB_QPT_UC:1303	case IB_QPT_UD:1304	case IB_QPT_XRC_INI:1305	case IB_QPT_XRC_TGT:1306	case IB_QPT_DRIVER:1307		break;1308	default:1309		return -EINVAL;1310	}1311 1312	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,1313						 &ib_dev);1314	if (IS_ERR(obj))1315		return PTR_ERR(obj);1316	obj->uxrcd = NULL;1317	obj->uevent.uobject.user_handle = cmd->user_handle;1318	mutex_init(&obj->mcast_lock);1319 1320	if (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE) {1321		ind_tbl = uobj_get_obj_read(rwq_ind_table,1322					    UVERBS_OBJECT_RWQ_IND_TBL,1323					    cmd->rwq_ind_tbl_handle, attrs);1324		if (!ind_tbl) {1325			ret = -EINVAL;1326			goto err_put;1327		}1328 1329		attr.rwq_ind_tbl = ind_tbl;1330	}1331 1332	if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) {1333		ret = -EINVAL;1334		goto err_put;1335	}1336 1337	if (ind_tbl && !cmd->max_send_wr)1338		has_sq = false;1339 1340	if (cmd->qp_type == IB_QPT_XRC_TGT) {1341		xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->pd_handle,1342					  attrs);1343 1344		if (IS_ERR(xrcd_uobj)) {1345			ret = -EINVAL;1346			goto err_put;1347		}1348 1349		xrcd = (struct ib_xrcd *)xrcd_uobj->object;1350		if (!xrcd) {1351			ret = -EINVAL;1352			goto err_put;1353		}1354		device = xrcd->device;1355	} else {1356		if (cmd->qp_type == IB_QPT_XRC_INI) {1357			cmd->max_recv_wr = 0;1358			cmd->max_recv_sge = 0;1359		} else {1360			if (cmd->is_srq) {1361				srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ,1362							cmd->srq_handle, attrs);1363				if (!srq || srq->srq_type == IB_SRQT_XRC) {1364					ret = -EINVAL;1365					goto err_put;1366				}1367			}1368 1369			if (!ind_tbl) {1370				if (cmd->recv_cq_handle != cmd->send_cq_handle) {1371					rcq = uobj_get_obj_read(1372						cq, UVERBS_OBJECT_CQ,1373						cmd->recv_cq_handle, attrs);1374					if (!rcq) {1375						ret = -EINVAL;1376						goto err_put;1377					}1378				}1379			}1380		}1381 1382		if (has_sq)1383			scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,1384						cmd->send_cq_handle, attrs);1385		if (!ind_tbl && cmd->qp_type != IB_QPT_XRC_INI)1386			rcq = rcq ?: scq;1387		pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle,1388				       attrs);1389		if (!pd || (!scq && has_sq)) {1390			ret = -EINVAL;1391			goto err_put;1392		}1393 1394		device = pd->device;1395	}1396 1397	attr.event_handler = ib_uverbs_qp_event_handler;1398	attr.send_cq       = scq;1399	attr.recv_cq       = rcq;1400	attr.srq           = srq;1401	attr.xrcd	   = xrcd;1402	attr.sq_sig_type   = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR :1403					      IB_SIGNAL_REQ_WR;1404	attr.qp_type       = cmd->qp_type;1405 1406	attr.cap.max_send_wr     = cmd->max_send_wr;1407	attr.cap.max_recv_wr     = cmd->max_recv_wr;1408	attr.cap.max_send_sge    = cmd->max_send_sge;1409	attr.cap.max_recv_sge    = cmd->max_recv_sge;1410	attr.cap.max_inline_data = cmd->max_inline_data;1411 1412	INIT_LIST_HEAD(&obj->uevent.event_list);1413	INIT_LIST_HEAD(&obj->mcast_list);1414 1415	attr.create_flags = cmd->create_flags;1416	if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |1417				IB_QP_CREATE_CROSS_CHANNEL |1418				IB_QP_CREATE_MANAGED_SEND |1419				IB_QP_CREATE_MANAGED_RECV |1420				IB_QP_CREATE_SCATTER_FCS |1421				IB_QP_CREATE_CVLAN_STRIPPING |1422				IB_QP_CREATE_SOURCE_QPN |1423				IB_QP_CREATE_PCI_WRITE_END_PADDING)) {1424		ret = -EINVAL;1425		goto err_put;1426	}1427 1428	if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) {1429		if (!capable(CAP_NET_RAW)) {1430			ret = -EPERM;1431			goto err_put;1432		}1433 1434		attr.source_qpn = cmd->source_qpn;1435	}1436 1437	qp = ib_create_qp_user(device, pd, &attr, &attrs->driver_udata, obj,1438			       KBUILD_MODNAME);1439	if (IS_ERR(qp)) {1440		ret = PTR_ERR(qp);1441		goto err_put;1442	}1443	ib_qp_usecnt_inc(qp);1444 1445	obj->uevent.uobject.object = qp;1446	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);1447	if (obj->uevent.event_file)1448		uverbs_uobject_get(&obj->uevent.event_file->uobj);1449 1450	if (xrcd) {1451		obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,1452					  uobject);1453		atomic_inc(&obj->uxrcd->refcnt);1454		uobj_put_read(xrcd_uobj);1455	}1456 1457	if (pd)1458		uobj_put_obj_read(pd);1459	if (scq)1460		rdma_lookup_put_uobject(&scq->uobject->uevent.uobject,1461					UVERBS_LOOKUP_READ);1462	if (rcq && rcq != scq)1463		rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject,1464					UVERBS_LOOKUP_READ);1465	if (srq)1466		rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,1467					UVERBS_LOOKUP_READ);1468	if (ind_tbl)1469		uobj_put_obj_read(ind_tbl);1470	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);1471 1472	resp.base.qpn             = qp->qp_num;1473	resp.base.qp_handle       = obj->uevent.uobject.id;1474	resp.base.max_recv_sge    = attr.cap.max_recv_sge;1475	resp.base.max_send_sge    = attr.cap.max_send_sge;1476	resp.base.max_recv_wr     = attr.cap.max_recv_wr;1477	resp.base.max_send_wr     = attr.cap.max_send_wr;1478	resp.base.max_inline_data = attr.cap.max_inline_data;1479	resp.response_length = uverbs_response_length(attrs, sizeof(resp));1480	return uverbs_response(attrs, &resp, sizeof(resp));1481 1482err_put:1483	if (!IS_ERR(xrcd_uobj))1484		uobj_put_read(xrcd_uobj);1485	if (pd)1486		uobj_put_obj_read(pd);1487	if (scq)1488		rdma_lookup_put_uobject(&scq->uobject->uevent.uobject,1489					UVERBS_LOOKUP_READ);1490	if (rcq && rcq != scq)1491		rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject,1492					UVERBS_LOOKUP_READ);1493	if (srq)1494		rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,1495					UVERBS_LOOKUP_READ);1496	if (ind_tbl)1497		uobj_put_obj_read(ind_tbl);1498 1499	uobj_alloc_abort(&obj->uevent.uobject, attrs);1500	return ret;1501}1502 1503static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs)1504{1505	struct ib_uverbs_create_qp      cmd;1506	struct ib_uverbs_ex_create_qp	cmd_ex;1507	int ret;1508 1509	ret = uverbs_request(attrs, &cmd, sizeof(cmd));1510	if (ret)1511		return ret;1512 1513	memset(&cmd_ex, 0, sizeof(cmd_ex));1514	cmd_ex.user_handle = cmd.user_handle;1515	cmd_ex.pd_handle = cmd.pd_handle;1516	cmd_ex.send_cq_handle = cmd.send_cq_handle;1517	cmd_ex.recv_cq_handle = cmd.recv_cq_handle;1518	cmd_ex.srq_handle = cmd.srq_handle;1519	cmd_ex.max_send_wr = cmd.max_send_wr;1520	cmd_ex.max_recv_wr = cmd.max_recv_wr;1521	cmd_ex.max_send_sge = cmd.max_send_sge;1522	cmd_ex.max_recv_sge = cmd.max_recv_sge;1523	cmd_ex.max_inline_data = cmd.max_inline_data;1524	cmd_ex.sq_sig_all = cmd.sq_sig_all;1525	cmd_ex.qp_type = cmd.qp_type;1526	cmd_ex.is_srq = cmd.is_srq;1527 1528	return create_qp(attrs, &cmd_ex);1529}1530 1531static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs)1532{1533	struct ib_uverbs_ex_create_qp cmd;1534	int ret;1535 1536	ret = uverbs_request(attrs, &cmd, sizeof(cmd));1537	if (ret)1538		return ret;1539 1540	if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK)1541		return -EINVAL;1542 1543	if (cmd.reserved)1544		return -EINVAL;1545 1546	return create_qp(attrs, &cmd);1547}1548 1549static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)1550{1551	struct ib_uverbs_create_qp_resp resp = {};1552	struct ib_uverbs_open_qp        cmd;1553	struct ib_uqp_object           *obj;1554	struct ib_xrcd		       *xrcd;1555	struct ib_qp                   *qp;1556	struct ib_qp_open_attr          attr = {};1557	int ret;1558	struct ib_uobject *xrcd_uobj;1559	struct ib_device *ib_dev;1560 1561	ret = uverbs_request(attrs, &cmd, sizeof(cmd));1562	if (ret)1563		return ret;1564 1565	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,1566						 &ib_dev);1567	if (IS_ERR(obj))1568		return PTR_ERR(obj);1569 1570	xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs);1571	if (IS_ERR(xrcd_uobj)) {1572		ret = -EINVAL;1573		goto err_put;1574	}1575 1576	xrcd = (struct ib_xrcd *)xrcd_uobj->object;1577	if (!xrcd) {1578		ret = -EINVAL;1579		goto err_xrcd;1580	}1581 1582	attr.event_handler = ib_uverbs_qp_event_handler;1583	attr.qp_num        = cmd.qpn;1584	attr.qp_type       = cmd.qp_type;1585 1586	INIT_LIST_HEAD(&obj->uevent.event_list);1587	INIT_LIST_HEAD(&obj->mcast_list);1588 1589	qp = ib_open_qp(xrcd, &attr);1590	if (IS_ERR(qp)) {1591		ret = PTR_ERR(qp);1592		goto err_xrcd;1593	}1594 1595	obj->uevent.uobject.object = qp;1596	obj->uevent.uobject.user_handle = cmd.user_handle;1597 1598	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);1599	atomic_inc(&obj->uxrcd->refcnt);1600	qp->uobject = obj;1601	uobj_put_read(xrcd_uobj);1602	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);1603 1604	resp.qpn = qp->qp_num;1605	resp.qp_handle = obj->uevent.uobject.id;1606	return uverbs_response(attrs, &resp, sizeof(resp));1607 1608err_xrcd:1609	uobj_put_read(xrcd_uobj);1610err_put:1611	uobj_alloc_abort(&obj->uevent.uobject, attrs);1612	return ret;1613}1614 1615static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr,1616				   struct rdma_ah_attr *rdma_attr)1617{1618	const struct ib_global_route   *grh;1619 1620	uverb_attr->dlid              = rdma_ah_get_dlid(rdma_attr);1621	uverb_attr->sl                = rdma_ah_get_sl(rdma_attr);1622	uverb_attr->src_path_bits     = rdma_ah_get_path_bits(rdma_attr);1623	uverb_attr->static_rate       = rdma_ah_get_static_rate(rdma_attr);1624	uverb_attr->is_global         = !!(rdma_ah_get_ah_flags(rdma_attr) &1625					 IB_AH_GRH);1626	if (uverb_attr->is_global) {1627		grh = rdma_ah_read_grh(rdma_attr);1628		memcpy(uverb_attr->dgid, grh->dgid.raw, 16);1629		uverb_attr->flow_label        = grh->flow_label;1630		uverb_attr->sgid_index        = grh->sgid_index;1631		uverb_attr->hop_limit         = grh->hop_limit;1632		uverb_attr->traffic_class     = grh->traffic_class;1633	}1634	uverb_attr->port_num          = rdma_ah_get_port_num(rdma_attr);1635}1636 1637static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs)1638{1639	struct ib_uverbs_query_qp      cmd;1640	struct ib_uverbs_query_qp_resp resp;1641	struct ib_qp                   *qp;1642	struct ib_qp_attr              *attr;1643	struct ib_qp_init_attr         *init_attr;1644	int                            ret;1645 1646	ret = uverbs_request(attrs, &cmd, sizeof(cmd));1647	if (ret)1648		return ret;1649 1650	attr      = kmalloc(sizeof *attr, GFP_KERNEL);1651	init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL);1652	if (!attr || !init_attr) {1653		ret = -ENOMEM;1654		goto out;1655	}1656 1657	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);1658	if (!qp) {1659		ret = -EINVAL;1660		goto out;1661	}1662 1663	ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr);1664 1665	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,1666				UVERBS_LOOKUP_READ);1667 1668	if (ret)1669		goto out;1670 1671	memset(&resp, 0, sizeof resp);1672 1673	resp.qp_state               = attr->qp_state;1674	resp.cur_qp_state           = attr->cur_qp_state;1675	resp.path_mtu               = attr->path_mtu;1676	resp.path_mig_state         = attr->path_mig_state;1677	resp.qkey                   = attr->qkey;1678	resp.rq_psn                 = attr->rq_psn;1679	resp.sq_psn                 = attr->sq_psn;1680	resp.dest_qp_num            = attr->dest_qp_num;1681	resp.qp_access_flags        = attr->qp_access_flags;1682	resp.pkey_index             = attr->pkey_index;1683	resp.alt_pkey_index         = attr->alt_pkey_index;1684	resp.sq_draining            = attr->sq_draining;1685	resp.max_rd_atomic          = attr->max_rd_atomic;1686	resp.max_dest_rd_atomic     = attr->max_dest_rd_atomic;1687	resp.min_rnr_timer          = attr->min_rnr_timer;1688	resp.port_num               = attr->port_num;1689	resp.timeout                = attr->timeout;1690	resp.retry_cnt              = attr->retry_cnt;1691	resp.rnr_retry              = attr->rnr_retry;1692	resp.alt_port_num           = attr->alt_port_num;1693	resp.alt_timeout            = attr->alt_timeout;1694 1695	copy_ah_attr_to_uverbs(&resp.dest, &attr->ah_attr);1696	copy_ah_attr_to_uverbs(&resp.alt_dest, &attr->alt_ah_attr);1697 1698	resp.max_send_wr            = init_attr->cap.max_send_wr;1699	resp.max_recv_wr            = init_attr->cap.max_recv_wr;1700	resp.max_send_sge           = init_attr->cap.max_send_sge;1701	resp.max_recv_sge           = init_attr->cap.max_recv_sge;1702	resp.max_inline_data        = init_attr->cap.max_inline_data;1703	resp.sq_sig_all             = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;1704 1705	ret = uverbs_response(attrs, &resp, sizeof(resp));1706 1707out:1708	kfree(attr);1709	kfree(init_attr);1710 1711	return ret;1712}1713 1714/* Remove ignored fields set in the attribute mask */1715static int modify_qp_mask(enum ib_qp_type qp_type, int mask)1716{1717	switch (qp_type) {1718	case IB_QPT_XRC_INI:1719		return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER);1720	case IB_QPT_XRC_TGT:1721		return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT |1722				IB_QP_RNR_RETRY);1723	default:1724		return mask;1725	}1726}1727 1728static void copy_ah_attr_from_uverbs(struct ib_device *dev,1729				     struct rdma_ah_attr *rdma_attr,1730				     struct ib_uverbs_qp_dest *uverb_attr)1731{1732	rdma_attr->type = rdma_ah_find_type(dev, uverb_attr->port_num);1733	if (uverb_attr->is_global) {1734		rdma_ah_set_grh(rdma_attr, NULL,1735				uverb_attr->flow_label,1736				uverb_attr->sgid_index,1737				uverb_attr->hop_limit,1738				uverb_attr->traffic_class);1739		rdma_ah_set_dgid_raw(rdma_attr, uverb_attr->dgid);1740	} else {1741		rdma_ah_set_ah_flags(rdma_attr, 0);1742	}1743	rdma_ah_set_dlid(rdma_attr, uverb_attr->dlid);1744	rdma_ah_set_sl(rdma_attr, uverb_attr->sl);1745	rdma_ah_set_path_bits(rdma_attr, uverb_attr->src_path_bits);1746	rdma_ah_set_static_rate(rdma_attr, uverb_attr->static_rate);1747	rdma_ah_set_port_num(rdma_attr, uverb_attr->port_num);1748	rdma_ah_set_make_grd(rdma_attr, false);1749}1750 1751static int modify_qp(struct uverbs_attr_bundle *attrs,1752		     struct ib_uverbs_ex_modify_qp *cmd)1753{1754	struct ib_qp_attr *attr;1755	struct ib_qp *qp;1756	int ret;1757 1758	attr = kzalloc(sizeof(*attr), GFP_KERNEL);1759	if (!attr)1760		return -ENOMEM;1761 1762	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle,1763			       attrs);1764	if (!qp) {1765		ret = -EINVAL;1766		goto out;1767	}1768 1769	if ((cmd->base.attr_mask & IB_QP_PORT) &&1770	    !rdma_is_port_valid(qp->device, cmd->base.port_num)) {1771		ret = -EINVAL;1772		goto release_qp;1773	}1774 1775	if ((cmd->base.attr_mask & IB_QP_AV)) {1776		if (!rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) {1777			ret = -EINVAL;1778			goto release_qp;1779		}1780 1781		if (cmd->base.attr_mask & IB_QP_STATE &&1782		    cmd->base.qp_state == IB_QPS_RTR) {1783		/* We are in INIT->RTR TRANSITION (if we are not,1784		 * this transition will be rejected in subsequent checks).1785		 * In the INIT->RTR transition, we cannot have IB_QP_PORT set,1786		 * but the IB_QP_STATE flag is required.1787		 *1788		 * Since kernel 3.14 (commit dbf727de7440), the uverbs driver,1789		 * when IB_QP_AV is set, has required inclusion of a valid1790		 * port number in the primary AV. (AVs are created and handled1791		 * differently for infiniband and ethernet (RoCE) ports).1792		 *1793		 * Check the port number included in the primary AV against1794		 * the port number in the qp struct, which was set (and saved)1795		 * in the RST->INIT transition.1796		 */1797			if (cmd->base.dest.port_num != qp->real_qp->port) {1798				ret = -EINVAL;1799				goto release_qp;1800			}1801		} else {1802		/* We are in SQD->SQD. (If we are not, this transition will1803		 * be rejected later in the verbs layer checks).1804		 * Check for both IB_QP_PORT and IB_QP_AV, these can be set1805		 * together in the SQD->SQD transition.1806		 *1807		 * If only IP_QP_AV was set, add in IB_QP_PORT as well (the1808		 * verbs layer driver does not track primary port changes1809		 * resulting from path migration. Thus, in SQD, if the primary1810		 * AV is modified, the primary port should also be modified).1811		 *1812		 * Note that in this transition, the IB_QP_STATE flag1813		 * is not allowed.1814		 */1815			if (((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))1816			     == (IB_QP_AV | IB_QP_PORT)) &&1817			    cmd->base.port_num != cmd->base.dest.port_num) {1818				ret = -EINVAL;1819				goto release_qp;1820			}1821			if ((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))1822			    == IB_QP_AV) {1823				cmd->base.attr_mask |= IB_QP_PORT;1824				cmd->base.port_num = cmd->base.dest.port_num;1825			}1826		}1827	}1828 1829	if ((cmd->base.attr_mask & IB_QP_ALT_PATH) &&1830	    (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) ||1831	    !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num) ||1832	    cmd->base.alt_port_num != cmd->base.alt_dest.port_num)) {1833		ret = -EINVAL;1834		goto release_qp;1835	}1836 1837	if ((cmd->base.attr_mask & IB_QP_CUR_STATE &&1838	    cmd->base.cur_qp_state > IB_QPS_ERR) ||1839	    (cmd->base.attr_mask & IB_QP_STATE &&1840	    cmd->base.qp_state > IB_QPS_ERR)) {1841		ret = -EINVAL;1842		goto release_qp;1843	}1844 1845	if (cmd->base.attr_mask & IB_QP_STATE)1846		attr->qp_state = cmd->base.qp_state;1847	if (cmd->base.attr_mask & IB_QP_CUR_STATE)1848		attr->cur_qp_state = cmd->base.cur_qp_state;1849	if (cmd->base.attr_mask & IB_QP_PATH_MTU)1850		attr->path_mtu = cmd->base.path_mtu;1851	if (cmd->base.attr_mask & IB_QP_PATH_MIG_STATE)1852		attr->path_mig_state = cmd->base.path_mig_state;1853	if (cmd->base.attr_mask & IB_QP_QKEY) {1854		if (cmd->base.qkey & IB_QP_SET_QKEY &&1855		    !rdma_nl_get_privileged_qkey()) {1856			ret = -EPERM;1857			goto release_qp;1858		}1859		attr->qkey = cmd->base.qkey;1860	}1861	if (cmd->base.attr_mask & IB_QP_RQ_PSN)1862		attr->rq_psn = cmd->base.rq_psn;1863	if (cmd->base.attr_mask & IB_QP_SQ_PSN)1864		attr->sq_psn = cmd->base.sq_psn;1865	if (cmd->base.attr_mask & IB_QP_DEST_QPN)1866		attr->dest_qp_num = cmd->base.dest_qp_num;1867	if (cmd->base.attr_mask & IB_QP_ACCESS_FLAGS)1868		attr->qp_access_flags = cmd->base.qp_access_flags;1869	if (cmd->base.attr_mask & IB_QP_PKEY_INDEX)1870		attr->pkey_index = cmd->base.pkey_index;1871	if (cmd->base.attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)1872		attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify;1873	if (cmd->base.attr_mask & IB_QP_MAX_QP_RD_ATOMIC)1874		attr->max_rd_atomic = cmd->base.max_rd_atomic;1875	if (cmd->base.attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)1876		attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic;1877	if (cmd->base.attr_mask & IB_QP_MIN_RNR_TIMER)1878		attr->min_rnr_timer = cmd->base.min_rnr_timer;1879	if (cmd->base.attr_mask & IB_QP_PORT)1880		attr->port_num = cmd->base.port_num;1881	if (cmd->base.attr_mask & IB_QP_TIMEOUT)1882		attr->timeout = cmd->base.timeout;1883	if (cmd->base.attr_mask & IB_QP_RETRY_CNT)1884		attr->retry_cnt = cmd->base.retry_cnt;1885	if (cmd->base.attr_mask & IB_QP_RNR_RETRY)1886		attr->rnr_retry = cmd->base.rnr_retry;1887	if (cmd->base.attr_mask & IB_QP_ALT_PATH) {1888		attr->alt_port_num = cmd->base.alt_port_num;1889		attr->alt_timeout = cmd->base.alt_timeout;1890		attr->alt_pkey_index = cmd->base.alt_pkey_index;1891	}1892	if (cmd->base.attr_mask & IB_QP_RATE_LIMIT)1893		attr->rate_limit = cmd->rate_limit;1894 1895	if (cmd->base.attr_mask & IB_QP_AV)1896		copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr,1897					 &cmd->base.dest);1898 1899	if (cmd->base.attr_mask & IB_QP_ALT_PATH)1900		copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr,1901					 &cmd->base.alt_dest);1902 1903	ret = ib_modify_qp_with_udata(qp, attr,1904				      modify_qp_mask(qp->qp_type,1905						     cmd->base.attr_mask),1906				      &attrs->driver_udata);1907 1908release_qp:1909	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,1910				UVERBS_LOOKUP_READ);1911out:1912	kfree(attr);1913 1914	return ret;1915}1916 1917static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs)1918{1919	struct ib_uverbs_ex_modify_qp cmd;1920	int ret;1921 1922	ret = uverbs_request(attrs, &cmd.base, sizeof(cmd.base));1923	if (ret)1924		return ret;1925 1926	if (cmd.base.attr_mask & ~IB_QP_ATTR_STANDARD_BITS)1927		return -EOPNOTSUPP;1928 1929	return modify_qp(attrs, &cmd);1930}1931 1932static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs)1933{1934	struct ib_uverbs_ex_modify_qp cmd;1935	struct ib_uverbs_ex_modify_qp_resp resp = {1936		.response_length = uverbs_response_length(attrs, sizeof(resp))1937	};1938	int ret;1939 1940	ret = uverbs_request(attrs, &cmd, sizeof(cmd));1941	if (ret)1942		return ret;1943 1944	/*1945	 * Last bit is reserved for extending the attr_mask by1946	 * using another field.1947	 */1948	if (cmd.base.attr_mask & ~(IB_QP_ATTR_STANDARD_BITS | IB_QP_RATE_LIMIT))1949		return -EOPNOTSUPP;1950 1951	ret = modify_qp(attrs, &cmd);1952	if (ret)1953		return ret;1954 1955	return uverbs_response(attrs, &resp, sizeof(resp));1956}1957 1958static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs)1959{1960	struct ib_uverbs_destroy_qp      cmd;1961	struct ib_uverbs_destroy_qp_resp resp;1962	struct ib_uobject		*uobj;1963	struct ib_uqp_object        	*obj;1964	int ret;1965 1966	ret = uverbs_request(attrs, &cmd, sizeof(cmd));1967	if (ret)1968		return ret;1969 1970	uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs);1971	if (IS_ERR(uobj))1972		return PTR_ERR(uobj);1973 1974	obj = container_of(uobj, struct ib_uqp_object, uevent.uobject);1975	memset(&resp, 0, sizeof(resp));1976	resp.events_reported = obj->uevent.events_reported;1977 1978	uobj_put_destroy(uobj);1979 1980	return uverbs_response(attrs, &resp, sizeof(resp));1981}1982 1983static void *alloc_wr(size_t wr_size, __u32 num_sge)1984{1985	if (num_sge >= (U32_MAX - ALIGN(wr_size, sizeof(struct ib_sge))) /1986			       sizeof(struct ib_sge))1987		return NULL;1988 1989	return kmalloc(ALIGN(wr_size, sizeof(struct ib_sge)) +1990			       num_sge * sizeof(struct ib_sge),1991		       GFP_KERNEL);1992}1993 1994static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)1995{1996	struct ib_uverbs_post_send      cmd;1997	struct ib_uverbs_post_send_resp resp;1998	struct ib_uverbs_send_wr       *user_wr;1999	struct ib_send_wr              *wr = NULL, *last, *next;2000	const struct ib_send_wr	       *bad_wr;2001	struct ib_qp                   *qp;2002	int                             i, sg_ind;2003	int				is_ud;2004	int ret, ret2;2005	size_t                          next_size;2006	const struct ib_sge __user *sgls;2007	const void __user *wqes;2008	struct uverbs_req_iter iter;2009 2010	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));2011	if (ret)2012		return ret;2013	wqes = uverbs_request_next_ptr(&iter, cmd.wqe_size * cmd.wr_count);2014	if (IS_ERR(wqes))2015		return PTR_ERR(wqes);2016	sgls = uverbs_request_next_ptr(2017		&iter, cmd.sge_count * sizeof(struct ib_uverbs_sge));2018	if (IS_ERR(sgls))2019		return PTR_ERR(sgls);2020	ret = uverbs_request_finish(&iter);2021	if (ret)2022		return ret;2023 2024	user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL);2025	if (!user_wr)2026		return -ENOMEM;2027 2028	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);2029	if (!qp) {2030		ret = -EINVAL;2031		goto out;2032	}2033 2034	is_ud = qp->qp_type == IB_QPT_UD;2035	sg_ind = 0;2036	last = NULL;2037	for (i = 0; i < cmd.wr_count; ++i) {2038		if (copy_from_user(user_wr, wqes + i * cmd.wqe_size,2039				   cmd.wqe_size)) {2040			ret = -EFAULT;2041			goto out_put;2042		}2043 2044		if (user_wr->num_sge + sg_ind > cmd.sge_count) {2045			ret = -EINVAL;2046			goto out_put;2047		}2048 2049		if (is_ud) {2050			struct ib_ud_wr *ud;2051 2052			if (user_wr->opcode != IB_WR_SEND &&2053			    user_wr->opcode != IB_WR_SEND_WITH_IMM) {2054				ret = -EINVAL;2055				goto out_put;2056			}2057 2058			next_size = sizeof(*ud);2059			ud = alloc_wr(next_size, user_wr->num_sge);2060			if (!ud) {2061				ret = -ENOMEM;2062				goto out_put;2063			}2064 2065			ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH,2066						   user_wr->wr.ud.ah, attrs);2067			if (!ud->ah) {2068				kfree(ud);2069				ret = -EINVAL;2070				goto out_put;2071			}2072			ud->remote_qpn = user_wr->wr.ud.remote_qpn;2073			ud->remote_qkey = user_wr->wr.ud.remote_qkey;2074 2075			next = &ud->wr;2076		} else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||2077			   user_wr->opcode == IB_WR_RDMA_WRITE ||2078			   user_wr->opcode == IB_WR_RDMA_READ) {2079			struct ib_rdma_wr *rdma;2080 2081			next_size = sizeof(*rdma);2082			rdma = alloc_wr(next_size, user_wr->num_sge);2083			if (!rdma) {2084				ret = -ENOMEM;2085				goto out_put;2086			}2087 2088			rdma->remote_addr = user_wr->wr.rdma.remote_addr;2089			rdma->rkey = user_wr->wr.rdma.rkey;2090 2091			next = &rdma->wr;2092		} else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||2093			   user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {2094			struct ib_atomic_wr *atomic;2095 2096			next_size = sizeof(*atomic);2097			atomic = alloc_wr(next_size, user_wr->num_sge);2098			if (!atomic) {2099				ret = -ENOMEM;2100				goto out_put;2101			}2102 2103			atomic->remote_addr = user_wr->wr.atomic.remote_addr;2104			atomic->compare_add = user_wr->wr.atomic.compare_add;2105			atomic->swap = user_wr->wr.atomic.swap;2106			atomic->rkey = user_wr->wr.atomic.rkey;2107 2108			next = &atomic->wr;2109		} else if (user_wr->opcode == IB_WR_SEND ||2110			   user_wr->opcode == IB_WR_SEND_WITH_IMM ||2111			   user_wr->opcode == IB_WR_SEND_WITH_INV) {2112			next_size = sizeof(*next);2113			next = alloc_wr(next_size, user_wr->num_sge);2114			if (!next) {2115				ret = -ENOMEM;2116				goto out_put;2117			}2118		} else {2119			ret = -EINVAL;2120			goto out_put;2121		}2122 2123		if (user_wr->opcode == IB_WR_SEND_WITH_IMM ||2124		    user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {2125			next->ex.imm_data =2126					(__be32 __force) user_wr->ex.imm_data;2127		} else if (user_wr->opcode == IB_WR_SEND_WITH_INV) {2128			next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey;2129		}2130 2131		if (!last)2132			wr = next;2133		else2134			last->next = next;2135		last = next;2136 2137		next->next       = NULL;2138		next->wr_id      = user_wr->wr_id;2139		next->num_sge    = user_wr->num_sge;2140		next->opcode     = user_wr->opcode;2141		next->send_flags = user_wr->send_flags;2142 2143		if (next->num_sge) {2144			next->sg_list = (void *) next +2145				ALIGN(next_size, sizeof(struct ib_sge));2146			if (copy_from_user(next->sg_list, sgls + sg_ind,2147					   next->num_sge *2148						   sizeof(struct ib_sge))) {2149				ret = -EFAULT;2150				goto out_put;2151			}2152			sg_ind += next->num_sge;2153		} else2154			next->sg_list = NULL;2155	}2156 2157	resp.bad_wr = 0;2158	ret = qp->device->ops.post_send(qp->real_qp, wr, &bad_wr);2159	if (ret)2160		for (next = wr; next; next = next->next) {2161			++resp.bad_wr;2162			if (next == bad_wr)2163				break;2164		}2165 2166	ret2 = uverbs_response(attrs, &resp, sizeof(resp));2167	if (ret2)2168		ret = ret2;2169 2170out_put:2171	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,2172				UVERBS_LOOKUP_READ);2173 2174	while (wr) {2175		if (is_ud && ud_wr(wr)->ah)2176			uobj_put_obj_read(ud_wr(wr)->ah);2177		next = wr->next;2178		kfree(wr);2179		wr = next;2180	}2181 2182out:2183	kfree(user_wr);2184 2185	return ret;2186}2187 2188static struct ib_recv_wr *2189ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count,2190			  u32 wqe_size, u32 sge_count)2191{2192	struct ib_uverbs_recv_wr *user_wr;2193	struct ib_recv_wr        *wr = NULL, *last, *next;2194	int                       sg_ind;2195	int                       i;2196	int                       ret;2197	const struct ib_sge __user *sgls;2198	const void __user *wqes;2199 2200	if (wqe_size < sizeof(struct ib_uverbs_recv_wr))2201		return ERR_PTR(-EINVAL);2202 2203	wqes = uverbs_request_next_ptr(iter, wqe_size * wr_count);2204	if (IS_ERR(wqes))2205		return ERR_CAST(wqes);2206	sgls = uverbs_request_next_ptr(2207		iter, sge_count * sizeof(struct ib_uverbs_sge));2208	if (IS_ERR(sgls))2209		return ERR_CAST(sgls);2210	ret = uverbs_request_finish(iter);2211	if (ret)2212		return ERR_PTR(ret);2213 2214	user_wr = kmalloc(wqe_size, GFP_KERNEL);2215	if (!user_wr)2216		return ERR_PTR(-ENOMEM);2217 2218	sg_ind = 0;2219	last = NULL;2220	for (i = 0; i < wr_count; ++i) {2221		if (copy_from_user(user_wr, wqes + i * wqe_size,2222				   wqe_size)) {2223			ret = -EFAULT;2224			goto err;2225		}2226 2227		if (user_wr->num_sge + sg_ind > sge_count) {2228			ret = -EINVAL;2229			goto err;2230		}2231 2232		if (user_wr->num_sge >=2233		    (U32_MAX - ALIGN(sizeof(*next), sizeof(struct ib_sge))) /2234			    sizeof(struct ib_sge)) {2235			ret = -EINVAL;2236			goto err;2237		}2238 2239		next = kmalloc(ALIGN(sizeof(*next), sizeof(struct ib_sge)) +2240				       user_wr->num_sge * sizeof(struct ib_sge),2241			       GFP_KERNEL);2242		if (!next) {2243			ret = -ENOMEM;2244			goto err;2245		}2246 2247		if (!last)2248			wr = next;2249		else2250			last->next = next;2251		last = next;2252 2253		next->next       = NULL;2254		next->wr_id      = user_wr->wr_id;2255		next->num_sge    = user_wr->num_sge;2256 2257		if (next->num_sge) {2258			next->sg_list = (void *)next +2259				ALIGN(sizeof(*next), sizeof(struct ib_sge));2260			if (copy_from_user(next->sg_list, sgls + sg_ind,2261					   next->num_sge *2262						   sizeof(struct ib_sge))) {2263				ret = -EFAULT;2264				goto err;2265			}2266			sg_ind += next->num_sge;2267		} else2268			next->sg_list = NULL;2269	}2270 2271	kfree(user_wr);2272	return wr;2273 2274err:2275	kfree(user_wr);2276 2277	while (wr) {2278		next = wr->next;2279		kfree(wr);2280		wr = next;2281	}2282 2283	return ERR_PTR(ret);2284}2285 2286static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs)2287{2288	struct ib_uverbs_post_recv      cmd;2289	struct ib_uverbs_post_recv_resp resp;2290	struct ib_recv_wr              *wr, *next;2291	const struct ib_recv_wr	       *bad_wr;2292	struct ib_qp                   *qp;2293	int ret, ret2;2294	struct uverbs_req_iter iter;2295 2296	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));2297	if (ret)2298		return ret;2299 2300	wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,2301				       cmd.sge_count);2302	if (IS_ERR(wr))2303		return PTR_ERR(wr);2304 2305	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);2306	if (!qp) {2307		ret = -EINVAL;2308		goto out;2309	}2310 2311	resp.bad_wr = 0;2312	ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr);2313 2314	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,2315				UVERBS_LOOKUP_READ);2316	if (ret) {2317		for (next = wr; next; next = next->next) {2318			++resp.bad_wr;2319			if (next == bad_wr)2320				break;2321		}2322	}2323 2324	ret2 = uverbs_response(attrs, &resp, sizeof(resp));2325	if (ret2)2326		ret = ret2;2327out:2328	while (wr) {2329		next = wr->next;2330		kfree(wr);2331		wr = next;2332	}2333 2334	return ret;2335}2336 2337static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs)2338{2339	struct ib_uverbs_post_srq_recv      cmd;2340	struct ib_uverbs_post_srq_recv_resp resp;2341	struct ib_recv_wr                  *wr, *next;2342	const struct ib_recv_wr		   *bad_wr;2343	struct ib_srq                      *srq;2344	int ret, ret2;2345	struct uverbs_req_iter iter;2346 2347	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));2348	if (ret)2349		return ret;2350 2351	wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,2352				       cmd.sge_count);2353	if (IS_ERR(wr))2354		return PTR_ERR(wr);2355 2356	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);2357	if (!srq) {2358		ret = -EINVAL;2359		goto out;2360	}2361 2362	resp.bad_wr = 0;2363	ret = srq->device->ops.post_srq_recv(srq, wr, &bad_wr);2364 2365	rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,2366				UVERBS_LOOKUP_READ);2367 2368	if (ret)2369		for (next = wr; next; next = next->next) {2370			++resp.bad_wr;2371			if (next == bad_wr)2372				break;2373		}2374 2375	ret2 = uverbs_response(attrs, &resp, sizeof(resp));2376	if (ret2)2377		ret = ret2;2378 2379out:2380	while (wr) {2381		next = wr->next;2382		kfree(wr);2383		wr = next;2384	}2385 2386	return ret;2387}2388 2389static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)2390{2391	struct ib_uverbs_create_ah	 cmd;2392	struct ib_uverbs_create_ah_resp	 resp;2393	struct ib_uobject		*uobj;2394	struct ib_pd			*pd;2395	struct ib_ah			*ah;2396	struct rdma_ah_attr		attr = {};2397	int ret;2398	struct ib_device *ib_dev;2399 2400	ret = uverbs_request(attrs, &cmd, sizeof(cmd));2401	if (ret)2402		return ret;2403 2404	uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev);2405	if (IS_ERR(uobj))2406		return PTR_ERR(uobj);2407 2408	if (!rdma_is_port_valid(ib_dev, cmd.attr.port_num)) {2409		ret = -EINVAL;2410		goto err;2411	}2412 2413	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);2414	if (!pd) {2415		ret = -EINVAL;2416		goto err;2417	}2418 2419	attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num);2420	rdma_ah_set_make_grd(&attr, false);2421	rdma_ah_set_dlid(&attr, cmd.attr.dlid);2422	rdma_ah_set_sl(&attr, cmd.attr.sl);2423	rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits);2424	rdma_ah_set_static_rate(&attr, cmd.attr.static_rate);2425	rdma_ah_set_port_num(&attr, cmd.attr.port_num);2426 2427	if (cmd.attr.is_global) {2428		rdma_ah_set_grh(&attr, NULL, cmd.attr.grh.flow_label,2429				cmd.attr.grh.sgid_index,2430				cmd.attr.grh.hop_limit,2431				cmd.attr.grh.traffic_class);2432		rdma_ah_set_dgid_raw(&attr, cmd.attr.grh.dgid);2433	} else {2434		rdma_ah_set_ah_flags(&attr, 0);2435	}2436 2437	ah = rdma_create_user_ah(pd, &attr, &attrs->driver_udata);2438	if (IS_ERR(ah)) {2439		ret = PTR_ERR(ah);2440		goto err_put;2441	}2442 2443	ah->uobject  = uobj;2444	uobj->user_handle = cmd.user_handle;2445	uobj->object = ah;2446	uobj_put_obj_read(pd);2447	uobj_finalize_uobj_create(uobj, attrs);2448 2449	resp.ah_handle = uobj->id;2450	return uverbs_response(attrs, &resp, sizeof(resp));2451 2452err_put:2453	uobj_put_obj_read(pd);2454err:2455	uobj_alloc_abort(uobj, attrs);2456	return ret;2457}2458 2459static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs)2460{2461	struct ib_uverbs_destroy_ah cmd;2462	int ret;2463 2464	ret = uverbs_request(attrs, &cmd, sizeof(cmd));2465	if (ret)2466		return ret;2467 2468	return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs);2469}2470 2471static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)2472{2473	struct ib_uverbs_attach_mcast cmd;2474	struct ib_qp                 *qp;2475	struct ib_uqp_object         *obj;2476	struct ib_uverbs_mcast_entry *mcast;2477	int                           ret;2478 2479	ret = uverbs_request(attrs, &cmd, sizeof(cmd));2480	if (ret)2481		return ret;2482 2483	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);2484	if (!qp)2485		return -EINVAL;2486 2487	obj = qp->uobject;2488 2489	mutex_lock(&obj->mcast_lock);2490	list_for_each_entry(mcast, &obj->mcast_list, list)2491		if (cmd.mlid == mcast->lid &&2492		    !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {2493			ret = 0;2494			goto out_put;2495		}2496 2497	mcast = kmalloc(sizeof *mcast, GFP_KERNEL);2498	if (!mcast) {2499		ret = -ENOMEM;2500		goto out_put;2501	}2502 2503	mcast->lid = cmd.mlid;2504	memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw);2505 2506	ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid);2507	if (!ret)2508		list_add_tail(&mcast->list, &obj->mcast_list);2509	else2510		kfree(mcast);2511 2512out_put:2513	mutex_unlock(&obj->mcast_lock);2514	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,2515				UVERBS_LOOKUP_READ);2516 2517	return ret;2518}2519 2520static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)2521{2522	struct ib_uverbs_detach_mcast cmd;2523	struct ib_uqp_object         *obj;2524	struct ib_qp                 *qp;2525	struct ib_uverbs_mcast_entry *mcast;2526	int                           ret;2527	bool                          found = false;2528 2529	ret = uverbs_request(attrs, &cmd, sizeof(cmd));2530	if (ret)2531		return ret;2532 2533	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);2534	if (!qp)2535		return -EINVAL;2536 2537	obj = qp->uobject;2538	mutex_lock(&obj->mcast_lock);2539 2540	list_for_each_entry(mcast, &obj->mcast_list, list)2541		if (cmd.mlid == mcast->lid &&2542		    !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {2543			list_del(&mcast->list);2544			kfree(mcast);2545			found = true;2546			break;2547		}2548 2549	if (!found) {2550		ret = -EINVAL;2551		goto out_put;2552	}2553 2554	ret = ib_detach_mcast(qp, (union ib_gid *)cmd.gid, cmd.mlid);2555 2556out_put:2557	mutex_unlock(&obj->mcast_lock);2558	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,2559				UVERBS_LOOKUP_READ);2560	return ret;2561}2562 2563struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)2564{2565	struct ib_uflow_resources *resources;2566 2567	resources = kzalloc(sizeof(*resources), GFP_KERNEL);2568 2569	if (!resources)2570		return NULL;2571 2572	if (!num_specs)2573		goto out;2574 2575	resources->counters =2576		kcalloc(num_specs, sizeof(*resources->counters), GFP_KERNEL);2577	resources->collection =2578		kcalloc(num_specs, sizeof(*resources->collection), GFP_KERNEL);2579 2580	if (!resources->counters || !resources->collection)2581		goto err;2582 2583out:2584	resources->max = num_specs;2585	return resources;2586 2587err:2588	kfree(resources->counters);2589	kfree(resources);2590 2591	return NULL;2592}2593EXPORT_SYMBOL(flow_resources_alloc);2594 2595void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res)2596{2597	unsigned int i;2598 2599	if (!uflow_res)2600		return;2601 2602	for (i = 0; i < uflow_res->collection_num; i++)2603		atomic_dec(&uflow_res->collection[i]->usecnt);2604 2605	for (i = 0; i < uflow_res->counters_num; i++)2606		atomic_dec(&uflow_res->counters[i]->usecnt);2607 2608	kfree(uflow_res->collection);2609	kfree(uflow_res->counters);2610	kfree(uflow_res);2611}2612EXPORT_SYMBOL(ib_uverbs_flow_resources_free);2613 2614void flow_resources_add(struct ib_uflow_resources *uflow_res,2615			enum ib_flow_spec_type type,2616			void *ibobj)2617{2618	WARN_ON(uflow_res->num >= uflow_res->max);2619 2620	switch (type) {2621	case IB_FLOW_SPEC_ACTION_HANDLE:2622		atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt);2623		uflow_res->collection[uflow_res->collection_num++] =2624			(struct ib_flow_action *)ibobj;2625		break;2626	case IB_FLOW_SPEC_ACTION_COUNT:2627		atomic_inc(&((struct ib_counters *)ibobj)->usecnt);2628		uflow_res->counters[uflow_res->counters_num++] =2629			(struct ib_counters *)ibobj;2630		break;2631	default:2632		WARN_ON(1);2633	}2634 2635	uflow_res->num++;2636}2637EXPORT_SYMBOL(flow_resources_add);2638 2639static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,2640				       struct ib_uverbs_flow_spec *kern_spec,2641				       union ib_flow_spec *ib_spec,2642				       struct ib_uflow_resources *uflow_res)2643{2644	ib_spec->type = kern_spec->type;2645	switch (ib_spec->type) {2646	case IB_FLOW_SPEC_ACTION_TAG:2647		if (kern_spec->flow_tag.size !=2648		    sizeof(struct ib_uverbs_flow_spec_action_tag))2649			return -EINVAL;2650 2651		ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag);2652		ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id;2653		break;2654	case IB_FLOW_SPEC_ACTION_DROP:2655		if (kern_spec->drop.size !=2656		    sizeof(struct ib_uverbs_flow_spec_action_drop))2657			return -EINVAL;2658 2659		ib_spec->drop.size = sizeof(struct ib_flow_spec_action_drop);2660		break;2661	case IB_FLOW_SPEC_ACTION_HANDLE:2662		if (kern_spec->action.size !=2663		    sizeof(struct ib_uverbs_flow_spec_action_handle))2664			return -EOPNOTSUPP;2665		ib_spec->action.act = uobj_get_obj_read(flow_action,2666							UVERBS_OBJECT_FLOW_ACTION,2667							kern_spec->action.handle,2668							attrs);2669		if (!ib_spec->action.act)2670			return -EINVAL;2671		ib_spec->action.size =2672			sizeof(struct ib_flow_spec_action_handle);2673		flow_resources_add(uflow_res,2674				   IB_FLOW_SPEC_ACTION_HANDLE,2675				   ib_spec->action.act);2676		uobj_put_obj_read(ib_spec->action.act);2677		break;2678	case IB_FLOW_SPEC_ACTION_COUNT:2679		if (kern_spec->flow_count.size !=2680			sizeof(struct ib_uverbs_flow_spec_action_count))2681			return -EINVAL;2682		ib_spec->flow_count.counters =2683			uobj_get_obj_read(counters,2684					  UVERBS_OBJECT_COUNTERS,2685					  kern_spec->flow_count.handle,2686					  attrs);2687		if (!ib_spec->flow_count.counters)2688			return -EINVAL;2689		ib_spec->flow_count.size =2690				sizeof(struct ib_flow_spec_action_count);2691		flow_resources_add(uflow_res,2692				   IB_FLOW_SPEC_ACTION_COUNT,2693				   ib_spec->flow_count.counters);2694		uobj_put_obj_read(ib_spec->flow_count.counters);2695		break;2696	default:2697		return -EINVAL;2698	}2699	return 0;2700}2701 2702static ssize_t spec_filter_size(const void *kern_spec_filter, u16 kern_filter_size,2703				u16 ib_real_filter_sz)2704{2705	/*2706	 * User space filter structures must be 64 bit aligned, otherwise this2707	 * may pass, but we won't handle additional new attributes.2708	 */2709 2710	if (kern_filter_size > ib_real_filter_sz) {2711		if (memchr_inv(kern_spec_filter +2712			       ib_real_filter_sz, 0,2713			       kern_filter_size - ib_real_filter_sz))2714			return -EINVAL;2715		return ib_real_filter_sz;2716	}2717	return kern_filter_size;2718}2719 2720int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,2721					  const void *kern_spec_mask,2722					  const void *kern_spec_val,2723					  size_t kern_filter_sz,2724					  union ib_flow_spec *ib_spec)2725{2726	ssize_t actual_filter_sz;2727	ssize_t ib_filter_sz;2728 2729	/* User flow spec size must be aligned to 4 bytes */2730	if (kern_filter_sz != ALIGN(kern_filter_sz, 4))2731		return -EINVAL;2732 2733	ib_spec->type = type;2734 2735	if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL))2736		return -EINVAL;2737 2738	switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {2739	case IB_FLOW_SPEC_ETH:2740		ib_filter_sz = sizeof(struct ib_flow_eth_filter);2741		actual_filter_sz = spec_filter_size(kern_spec_mask,2742						    kern_filter_sz,2743						    ib_filter_sz);2744		if (actual_filter_sz <= 0)2745			return -EINVAL;2746		ib_spec->size = sizeof(struct ib_flow_spec_eth);2747		memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz);2748		memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz);2749		break;2750	case IB_FLOW_SPEC_IPV4:2751		ib_filter_sz = sizeof(struct ib_flow_ipv4_filter);2752		actual_filter_sz = spec_filter_size(kern_spec_mask,2753						    kern_filter_sz,2754						    ib_filter_sz);2755		if (actual_filter_sz <= 0)2756			return -EINVAL;2757		ib_spec->size = sizeof(struct ib_flow_spec_ipv4);2758		memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz);2759		memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz);2760		break;2761	case IB_FLOW_SPEC_IPV6:2762		ib_filter_sz = sizeof(struct ib_flow_ipv6_filter);2763		actual_filter_sz = spec_filter_size(kern_spec_mask,2764						    kern_filter_sz,2765						    ib_filter_sz);2766		if (actual_filter_sz <= 0)2767			return -EINVAL;2768		ib_spec->size = sizeof(struct ib_flow_spec_ipv6);2769		memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz);2770		memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz);2771 2772		if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) ||2773		    (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20))2774			return -EINVAL;2775		break;2776	case IB_FLOW_SPEC_TCP:2777	case IB_FLOW_SPEC_UDP:2778		ib_filter_sz = sizeof(struct ib_flow_tcp_udp_filter);2779		actual_filter_sz = spec_filter_size(kern_spec_mask,2780						    kern_filter_sz,2781						    ib_filter_sz);2782		if (actual_filter_sz <= 0)2783			return -EINVAL;2784		ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp);2785		memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz);2786		memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz);2787		break;2788	case IB_FLOW_SPEC_VXLAN_TUNNEL:2789		ib_filter_sz = sizeof(struct ib_flow_tunnel_filter);2790		actual_filter_sz = spec_filter_size(kern_spec_mask,2791						    kern_filter_sz,2792						    ib_filter_sz);2793		if (actual_filter_sz <= 0)2794			return -EINVAL;2795		ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel);2796		memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz);2797		memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz);2798 2799		if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) ||2800		    (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24))2801			return -EINVAL;2802		break;2803	case IB_FLOW_SPEC_ESP:2804		ib_filter_sz = sizeof(struct ib_flow_esp_filter);2805		actual_filter_sz = spec_filter_size(kern_spec_mask,2806						    kern_filter_sz,2807						    ib_filter_sz);2808		if (actual_filter_sz <= 0)2809			return -EINVAL;2810		ib_spec->esp.size = sizeof(struct ib_flow_spec_esp);2811		memcpy(&ib_spec->esp.val, kern_spec_val, actual_filter_sz);2812		memcpy(&ib_spec->esp.mask, kern_spec_mask, actual_filter_sz);2813		break;2814	case IB_FLOW_SPEC_GRE:2815		ib_filter_sz = sizeof(struct ib_flow_gre_filter);2816		actual_filter_sz = spec_filter_size(kern_spec_mask,2817						    kern_filter_sz,2818						    ib_filter_sz);2819		if (actual_filter_sz <= 0)2820			return -EINVAL;2821		ib_spec->gre.size = sizeof(struct ib_flow_spec_gre);2822		memcpy(&ib_spec->gre.val, kern_spec_val, actual_filter_sz);2823		memcpy(&ib_spec->gre.mask, kern_spec_mask, actual_filter_sz);2824		break;2825	case IB_FLOW_SPEC_MPLS:2826		ib_filter_sz = sizeof(struct ib_flow_mpls_filter);2827		actual_filter_sz = spec_filter_size(kern_spec_mask,2828						    kern_filter_sz,2829						    ib_filter_sz);2830		if (actual_filter_sz <= 0)2831			return -EINVAL;2832		ib_spec->mpls.size = sizeof(struct ib_flow_spec_mpls);2833		memcpy(&ib_spec->mpls.val, kern_spec_val, actual_filter_sz);2834		memcpy(&ib_spec->mpls.mask, kern_spec_mask, actual_filter_sz);2835		break;2836	default:2837		return -EINVAL;2838	}2839	return 0;2840}2841 2842static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec,2843				       union ib_flow_spec *ib_spec)2844{2845	size_t kern_filter_sz;2846	void *kern_spec_mask;2847	void *kern_spec_val;2848 2849	if (check_sub_overflow((size_t)kern_spec->hdr.size,2850			       sizeof(struct ib_uverbs_flow_spec_hdr),2851			       &kern_filter_sz))2852		return -EINVAL;2853 2854	kern_filter_sz /= 2;2855 2856	kern_spec_val = (void *)kern_spec +2857		sizeof(struct ib_uverbs_flow_spec_hdr);2858	kern_spec_mask = kern_spec_val + kern_filter_sz;2859 2860	return ib_uverbs_kern_spec_to_ib_spec_filter(kern_spec->type,2861						     kern_spec_mask,2862						     kern_spec_val,2863						     kern_filter_sz, ib_spec);2864}2865 2866static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs,2867				struct ib_uverbs_flow_spec *kern_spec,2868				union ib_flow_spec *ib_spec,2869				struct ib_uflow_resources *uflow_res)2870{2871	if (kern_spec->reserved)2872		return -EINVAL;2873 2874	if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG)2875		return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec,2876						   uflow_res);2877	else2878		return kern_spec_to_ib_spec_filter(kern_spec, ib_spec);2879}2880 2881static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)2882{2883	struct ib_uverbs_ex_create_wq cmd;2884	struct ib_uverbs_ex_create_wq_resp resp = {};2885	struct ib_uwq_object           *obj;2886	int err = 0;2887	struct ib_cq *cq;2888	struct ib_pd *pd;2889	struct ib_wq *wq;2890	struct ib_wq_init_attr wq_init_attr = {};2891	struct ib_device *ib_dev;2892 2893	err = uverbs_request(attrs, &cmd, sizeof(cmd));2894	if (err)2895		return err;2896 2897	if (cmd.comp_mask)2898		return -EOPNOTSUPP;2899 2900	obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs,2901						 &ib_dev);2902	if (IS_ERR(obj))2903		return PTR_ERR(obj);2904 2905	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);2906	if (!pd) {2907		err = -EINVAL;2908		goto err_uobj;2909	}2910 2911	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);2912	if (!cq) {2913		err = -EINVAL;2914		goto err_put_pd;2915	}2916 2917	wq_init_attr.cq = cq;2918	wq_init_attr.max_sge = cmd.max_sge;2919	wq_init_attr.max_wr = cmd.max_wr;2920	wq_init_attr.wq_type = cmd.wq_type;2921	wq_init_attr.event_handler = ib_uverbs_wq_event_handler;2922	wq_init_attr.create_flags = cmd.create_flags;2923	INIT_LIST_HEAD(&obj->uevent.event_list);2924	obj->uevent.uobject.user_handle = cmd.user_handle;2925 2926	wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);2927	if (IS_ERR(wq)) {2928		err = PTR_ERR(wq);2929		goto err_put_cq;2930	}2931 2932	wq->uobject = obj;2933	obj->uevent.uobject.object = wq;2934	wq->wq_type = wq_init_attr.wq_type;2935	wq->cq = cq;2936	wq->pd = pd;2937	wq->device = pd->device;2938	atomic_set(&wq->usecnt, 0);2939	atomic_inc(&pd->usecnt);2940	atomic_inc(&cq->usecnt);2941	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);2942	if (obj->uevent.event_file)2943		uverbs_uobject_get(&obj->uevent.event_file->uobj);2944 2945	uobj_put_obj_read(pd);2946	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,2947				UVERBS_LOOKUP_READ);2948	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);2949 2950	resp.wq_handle = obj->uevent.uobject.id;2951	resp.max_sge = wq_init_attr.max_sge;2952	resp.max_wr = wq_init_attr.max_wr;2953	resp.wqn = wq->wq_num;2954	resp.response_length = uverbs_response_length(attrs, sizeof(resp));2955	return uverbs_response(attrs, &resp, sizeof(resp));2956 2957err_put_cq:2958	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,2959				UVERBS_LOOKUP_READ);2960err_put_pd:2961	uobj_put_obj_read(pd);2962err_uobj:2963	uobj_alloc_abort(&obj->uevent.uobject, attrs);2964 2965	return err;2966}2967 2968static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs)2969{2970	struct ib_uverbs_ex_destroy_wq	cmd;2971	struct ib_uverbs_ex_destroy_wq_resp	resp = {};2972	struct ib_uobject		*uobj;2973	struct ib_uwq_object		*obj;2974	int				ret;2975 2976	ret = uverbs_request(attrs, &cmd, sizeof(cmd));2977	if (ret)2978		return ret;2979 2980	if (cmd.comp_mask)2981		return -EOPNOTSUPP;2982 2983	resp.response_length = uverbs_response_length(attrs, sizeof(resp));2984	uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);2985	if (IS_ERR(uobj))2986		return PTR_ERR(uobj);2987 2988	obj = container_of(uobj, struct ib_uwq_object, uevent.uobject);2989	resp.events_reported = obj->uevent.events_reported;2990 2991	uobj_put_destroy(uobj);2992 2993	return uverbs_response(attrs, &resp, sizeof(resp));2994}2995 2996static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs)2997{2998	struct ib_uverbs_ex_modify_wq cmd;2999	struct ib_wq *wq;3000	struct ib_wq_attr wq_attr = {};3001	int ret;3002 3003	ret = uverbs_request(attrs, &cmd, sizeof(cmd));3004	if (ret)3005		return ret;3006 3007	if (!cmd.attr_mask)3008		return -EINVAL;3009 3010	if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS))3011		return -EINVAL;3012 3013	wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);3014	if (!wq)3015		return -EINVAL;3016 3017	if (cmd.attr_mask & IB_WQ_FLAGS) {3018		wq_attr.flags = cmd.flags;3019		wq_attr.flags_mask = cmd.flags_mask;3020	}3021 3022	if (cmd.attr_mask & IB_WQ_CUR_STATE) {3023		if (cmd.curr_wq_state > IB_WQS_ERR)3024			return -EINVAL;3025 3026		wq_attr.curr_wq_state = cmd.curr_wq_state;3027	} else {3028		wq_attr.curr_wq_state = wq->state;3029	}3030 3031	if (cmd.attr_mask & IB_WQ_STATE) {3032		if (cmd.wq_state > IB_WQS_ERR)3033			return -EINVAL;3034 3035		wq_attr.wq_state = cmd.wq_state;3036	} else {3037		wq_attr.wq_state = wq_attr.curr_wq_state;3038	}3039 3040	ret = wq->device->ops.modify_wq(wq, &wq_attr, cmd.attr_mask,3041					&attrs->driver_udata);3042	rdma_lookup_put_uobject(&wq->uobject->uevent.uobject,3043				UVERBS_LOOKUP_READ);3044	return ret;3045}3046 3047static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)3048{3049	struct ib_uverbs_ex_create_rwq_ind_table cmd;3050	struct ib_uverbs_ex_create_rwq_ind_table_resp  resp = {};3051	struct ib_uobject *uobj;3052	int err;3053	struct ib_rwq_ind_table_init_attr init_attr = {};3054	struct ib_rwq_ind_table *rwq_ind_tbl;3055	struct ib_wq **wqs = NULL;3056	u32 *wqs_handles = NULL;3057	struct ib_wq	*wq = NULL;3058	int i, num_read_wqs;3059	u32 num_wq_handles;3060	struct uverbs_req_iter iter;3061	struct ib_device *ib_dev;3062 3063	err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));3064	if (err)3065		return err;3066 3067	if (cmd.comp_mask)3068		return -EOPNOTSUPP;3069 3070	if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE)3071		return -EINVAL;3072 3073	num_wq_handles = 1 << cmd.log_ind_tbl_size;3074	wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles),3075			      GFP_KERNEL);3076	if (!wqs_handles)3077		return -ENOMEM;3078 3079	err = uverbs_request_next(&iter, wqs_handles,3080				  num_wq_handles * sizeof(__u32));3081	if (err)3082		goto err_free;3083 3084	err = uverbs_request_finish(&iter);3085	if (err)3086		goto err_free;3087 3088	wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL);3089	if (!wqs) {3090		err = -ENOMEM;3091		goto  err_free;3092	}3093 3094	for (num_read_wqs = 0; num_read_wqs < num_wq_handles;3095			num_read_wqs++) {3096		wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ,3097				       wqs_handles[num_read_wqs], attrs);3098		if (!wq) {3099			err = -EINVAL;3100			goto put_wqs;3101		}3102 3103		wqs[num_read_wqs] = wq;3104		atomic_inc(&wqs[num_read_wqs]->usecnt);3105	}3106 3107	uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);3108	if (IS_ERR(uobj)) {3109		err = PTR_ERR(uobj);3110		goto put_wqs;3111	}3112 3113	rwq_ind_tbl = rdma_zalloc_drv_obj(ib_dev, ib_rwq_ind_table);3114	if (!rwq_ind_tbl) {3115		err = -ENOMEM;3116		goto err_uobj;3117	}3118 3119	init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;3120	init_attr.ind_tbl = wqs;3121 3122	rwq_ind_tbl->ind_tbl = wqs;3123	rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size;3124	rwq_ind_tbl->uobject = uobj;3125	uobj->object = rwq_ind_tbl;3126	rwq_ind_tbl->device = ib_dev;3127	atomic_set(&rwq_ind_tbl->usecnt, 0);3128 3129	err = ib_dev->ops.create_rwq_ind_table(rwq_ind_tbl, &init_attr,3130					       &attrs->driver_udata);3131	if (err)3132		goto err_create;3133 3134	for (i = 0; i < num_wq_handles; i++)3135		rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject,3136					UVERBS_LOOKUP_READ);3137	kfree(wqs_handles);3138	uobj_finalize_uobj_create(uobj, attrs);3139 3140	resp.ind_tbl_handle = uobj->id;3141	resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;3142	resp.response_length = uverbs_response_length(attrs, sizeof(resp));3143	return uverbs_response(attrs, &resp, sizeof(resp));3144 3145err_create:3146	kfree(rwq_ind_tbl);3147err_uobj:3148	uobj_alloc_abort(uobj, attrs);3149put_wqs:3150	for (i = 0; i < num_read_wqs; i++) {3151		rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject,3152					UVERBS_LOOKUP_READ);3153		atomic_dec(&wqs[i]->usecnt);3154	}3155err_free:3156	kfree(wqs_handles);3157	kfree(wqs);3158	return err;3159}3160 3161static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs)3162{3163	struct ib_uverbs_ex_destroy_rwq_ind_table cmd;3164	int ret;3165 3166	ret = uverbs_request(attrs, &cmd, sizeof(cmd));3167	if (ret)3168		return ret;3169 3170	if (cmd.comp_mask)3171		return -EOPNOTSUPP;3172 3173	return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL,3174				    cmd.ind_tbl_handle, attrs);3175}3176 3177static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)3178{3179	struct ib_uverbs_create_flow	  cmd;3180	struct ib_uverbs_create_flow_resp resp = {};3181	struct ib_uobject		  *uobj;3182	struct ib_flow			  *flow_id;3183	struct ib_uverbs_flow_attr	  *kern_flow_attr;3184	struct ib_flow_attr		  *flow_attr;3185	struct ib_qp			  *qp;3186	struct ib_uflow_resources	  *uflow_res;3187	struct ib_uverbs_flow_spec_hdr	  *kern_spec;3188	struct uverbs_req_iter iter;3189	int err;3190	void *ib_spec;3191	int i;3192	struct ib_device *ib_dev;3193 3194	err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));3195	if (err)3196		return err;3197 3198	if (cmd.comp_mask)3199		return -EINVAL;3200 3201	if (!capable(CAP_NET_RAW))3202		return -EPERM;3203 3204	if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED)3205		return -EINVAL;3206 3207	if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) &&3208	    ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) ||3209	     (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT)))3210		return -EINVAL;3211 3212	if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)3213		return -EINVAL;3214 3215	if (cmd.flow_attr.size >3216	    (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec)))3217		return -EINVAL;3218 3219	if (cmd.flow_attr.reserved[0] ||3220	    cmd.flow_attr.reserved[1])3221		return -EINVAL;3222 3223	if (cmd.flow_attr.num_of_specs) {3224		kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size,3225					 GFP_KERNEL);3226		if (!kern_flow_attr)3227			return -ENOMEM;3228 3229		*kern_flow_attr = cmd.flow_attr;3230		err = uverbs_request_next(&iter, &kern_flow_attr->flow_specs,3231					  cmd.flow_attr.size);3232		if (err)3233			goto err_free_attr;3234	} else {3235		kern_flow_attr = &cmd.flow_attr;3236	}3237 3238	err = uverbs_request_finish(&iter);3239	if (err)3240		goto err_free_attr;3241 3242	uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev);3243	if (IS_ERR(uobj)) {3244		err = PTR_ERR(uobj);3245		goto err_free_attr;3246	}3247 3248	if (!rdma_is_port_valid(uobj->context->device, cmd.flow_attr.port)) {3249		err = -EINVAL;3250		goto err_uobj;3251	}3252 3253	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);3254	if (!qp) {3255		err = -EINVAL;3256		goto err_uobj;3257	}3258 3259	if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) {3260		err = -EINVAL;3261		goto err_put;3262	}3263 3264	flow_attr = kzalloc(struct_size(flow_attr, flows,3265				cmd.flow_attr.num_of_specs), GFP_KERNEL);3266	if (!flow_attr) {3267		err = -ENOMEM;3268		goto err_put;3269	}3270	uflow_res = flow_resources_alloc(cmd.flow_attr.num_of_specs);3271	if (!uflow_res) {3272		err = -ENOMEM;3273		goto err_free_flow_attr;3274	}3275 3276	flow_attr->type = kern_flow_attr->type;3277	flow_attr->priority = kern_flow_attr->priority;3278	flow_attr->num_of_specs = kern_flow_attr->num_of_specs;3279	flow_attr->port = kern_flow_attr->port;3280	flow_attr->flags = kern_flow_attr->flags;3281	flow_attr->size = sizeof(*flow_attr);3282 3283	kern_spec = kern_flow_attr->flow_specs;3284	ib_spec = flow_attr + 1;3285	for (i = 0; i < flow_attr->num_of_specs &&3286			cmd.flow_attr.size >= sizeof(*kern_spec) &&3287			cmd.flow_attr.size >= kern_spec->size;3288	     i++) {3289		err = kern_spec_to_ib_spec(3290				attrs, (struct ib_uverbs_flow_spec *)kern_spec,3291				ib_spec, uflow_res);3292		if (err)3293			goto err_free;3294 3295		flow_attr->size +=3296			((union ib_flow_spec *) ib_spec)->size;3297		cmd.flow_attr.size -= kern_spec->size;3298		kern_spec = ((void *)kern_spec) + kern_spec->size;3299		ib_spec += ((union ib_flow_spec *) ib_spec)->size;3300	}3301	if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) {3302		pr_warn("create flow failed, flow %d: %u bytes left from uverb cmd\n",3303			i, cmd.flow_attr.size);3304		err = -EINVAL;3305		goto err_free;3306	}3307 3308	flow_id = qp->device->ops.create_flow(qp, flow_attr,3309					      &attrs->driver_udata);3310 3311	if (IS_ERR(flow_id)) {3312		err = PTR_ERR(flow_id);3313		goto err_free;3314	}3315 3316	ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res);3317 3318	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,3319				UVERBS_LOOKUP_READ);3320	kfree(flow_attr);3321 3322	if (cmd.flow_attr.num_of_specs)3323		kfree(kern_flow_attr);3324	uobj_finalize_uobj_create(uobj, attrs);3325 3326	resp.flow_handle = uobj->id;3327	return uverbs_response(attrs, &resp, sizeof(resp));3328 3329err_free:3330	ib_uverbs_flow_resources_free(uflow_res);3331err_free_flow_attr:3332	kfree(flow_attr);3333err_put:3334	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,3335				UVERBS_LOOKUP_READ);3336err_uobj:3337	uobj_alloc_abort(uobj, attrs);3338err_free_attr:3339	if (cmd.flow_attr.num_of_specs)3340		kfree(kern_flow_attr);3341	return err;3342}3343 3344static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs)3345{3346	struct ib_uverbs_destroy_flow	cmd;3347	int				ret;3348 3349	ret = uverbs_request(attrs, &cmd, sizeof(cmd));3350	if (ret)3351		return ret;3352 3353	if (cmd.comp_mask)3354		return -EINVAL;3355 3356	return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs);3357}3358 3359static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,3360				struct ib_uverbs_create_xsrq *cmd,3361				struct ib_udata *udata)3362{3363	struct ib_uverbs_create_srq_resp resp = {};3364	struct ib_usrq_object           *obj;3365	struct ib_pd                    *pd;3366	struct ib_srq                   *srq;3367	struct ib_srq_init_attr          attr;3368	int ret;3369	struct ib_uobject *xrcd_uobj;3370	struct ib_device *ib_dev;3371 3372	obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,3373						  &ib_dev);3374	if (IS_ERR(obj))3375		return PTR_ERR(obj);3376 3377	if (cmd->srq_type == IB_SRQT_TM)3378		attr.ext.tag_matching.max_num_tags = cmd->max_num_tags;3379 3380	if (cmd->srq_type == IB_SRQT_XRC) {3381		xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->xrcd_handle,3382					  attrs);3383		if (IS_ERR(xrcd_uobj)) {3384			ret = -EINVAL;3385			goto err;3386		}3387 3388		attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object;3389		if (!attr.ext.xrc.xrcd) {3390			ret = -EINVAL;3391			goto err_put_xrcd;3392		}3393 3394		obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);3395		atomic_inc(&obj->uxrcd->refcnt);3396	}3397 3398	if (ib_srq_has_cq(cmd->srq_type)) {3399		attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,3400						cmd->cq_handle, attrs);3401		if (!attr.ext.cq) {3402			ret = -EINVAL;3403			goto err_put_xrcd;3404		}3405	}3406 3407	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs);3408	if (!pd) {3409		ret = -EINVAL;3410		goto err_put_cq;3411	}3412 3413	attr.event_handler  = ib_uverbs_srq_event_handler;3414	attr.srq_type       = cmd->srq_type;3415	attr.attr.max_wr    = cmd->max_wr;3416	attr.attr.max_sge   = cmd->max_sge;3417	attr.attr.srq_limit = cmd->srq_limit;3418 3419	INIT_LIST_HEAD(&obj->uevent.event_list);3420	obj->uevent.uobject.user_handle = cmd->user_handle;3421 3422	srq = ib_create_srq_user(pd, &attr, obj, udata);3423	if (IS_ERR(srq)) {3424		ret = PTR_ERR(srq);3425		goto err_put_pd;3426	}3427 3428	obj->uevent.uobject.object = srq;3429	obj->uevent.uobject.user_handle = cmd->user_handle;3430	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);3431	if (obj->uevent.event_file)3432		uverbs_uobject_get(&obj->uevent.event_file->uobj);3433 3434	if (cmd->srq_type == IB_SRQT_XRC)3435		resp.srqn = srq->ext.xrc.srq_num;3436 3437	if (cmd->srq_type == IB_SRQT_XRC)3438		uobj_put_read(xrcd_uobj);3439 3440	if (ib_srq_has_cq(cmd->srq_type))3441		rdma_lookup_put_uobject(&attr.ext.cq->uobject->uevent.uobject,3442					UVERBS_LOOKUP_READ);3443 3444	uobj_put_obj_read(pd);3445	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);3446 3447	resp.srq_handle = obj->uevent.uobject.id;3448	resp.max_wr = attr.attr.max_wr;3449	resp.max_sge = attr.attr.max_sge;3450	return uverbs_response(attrs, &resp, sizeof(resp));3451 3452err_put_pd:3453	uobj_put_obj_read(pd);3454err_put_cq:3455	if (ib_srq_has_cq(cmd->srq_type))3456		rdma_lookup_put_uobject(&attr.ext.cq->uobject->uevent.uobject,3457					UVERBS_LOOKUP_READ);3458 3459err_put_xrcd:3460	if (cmd->srq_type == IB_SRQT_XRC) {3461		atomic_dec(&obj->uxrcd->refcnt);3462		uobj_put_read(xrcd_uobj);3463	}3464 3465err:3466	uobj_alloc_abort(&obj->uevent.uobject, attrs);3467	return ret;3468}3469 3470static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs)3471{3472	struct ib_uverbs_create_srq      cmd;3473	struct ib_uverbs_create_xsrq     xcmd;3474	int ret;3475 3476	ret = uverbs_request(attrs, &cmd, sizeof(cmd));3477	if (ret)3478		return ret;3479 3480	memset(&xcmd, 0, sizeof(xcmd));3481	xcmd.response	 = cmd.response;3482	xcmd.user_handle = cmd.user_handle;3483	xcmd.srq_type	 = IB_SRQT_BASIC;3484	xcmd.pd_handle	 = cmd.pd_handle;3485	xcmd.max_wr	 = cmd.max_wr;3486	xcmd.max_sge	 = cmd.max_sge;3487	xcmd.srq_limit	 = cmd.srq_limit;3488 3489	return __uverbs_create_xsrq(attrs, &xcmd, &attrs->driver_udata);3490}3491 3492static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs)3493{3494	struct ib_uverbs_create_xsrq     cmd;3495	int ret;3496 3497	ret = uverbs_request(attrs, &cmd, sizeof(cmd));3498	if (ret)3499		return ret;3500 3501	return __uverbs_create_xsrq(attrs, &cmd, &attrs->driver_udata);3502}3503 3504static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)3505{3506	struct ib_uverbs_modify_srq cmd;3507	struct ib_srq              *srq;3508	struct ib_srq_attr          attr;3509	int                         ret;3510 3511	ret = uverbs_request(attrs, &cmd, sizeof(cmd));3512	if (ret)3513		return ret;3514 3515	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);3516	if (!srq)3517		return -EINVAL;3518 3519	attr.max_wr    = cmd.max_wr;3520	attr.srq_limit = cmd.srq_limit;3521 3522	ret = srq->device->ops.modify_srq(srq, &attr, cmd.attr_mask,3523					  &attrs->driver_udata);3524 3525	rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,3526				UVERBS_LOOKUP_READ);3527 3528	return ret;3529}3530 3531static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs)3532{3533	struct ib_uverbs_query_srq      cmd;3534	struct ib_uverbs_query_srq_resp resp;3535	struct ib_srq_attr              attr;3536	struct ib_srq                   *srq;3537	int                             ret;3538 3539	ret = uverbs_request(attrs, &cmd, sizeof(cmd));3540	if (ret)3541		return ret;3542 3543	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);3544	if (!srq)3545		return -EINVAL;3546 3547	ret = ib_query_srq(srq, &attr);3548 3549	rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,3550				UVERBS_LOOKUP_READ);3551 3552	if (ret)3553		return ret;3554 3555	memset(&resp, 0, sizeof resp);3556 3557	resp.max_wr    = attr.max_wr;3558	resp.max_sge   = attr.max_sge;3559	resp.srq_limit = attr.srq_limit;3560 3561	return uverbs_response(attrs, &resp, sizeof(resp));3562}3563 3564static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs)3565{3566	struct ib_uverbs_destroy_srq      cmd;3567	struct ib_uverbs_destroy_srq_resp resp;3568	struct ib_uobject		 *uobj;3569	struct ib_uevent_object        	 *obj;3570	int ret;3571 3572	ret = uverbs_request(attrs, &cmd, sizeof(cmd));3573	if (ret)3574		return ret;3575 3576	uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);3577	if (IS_ERR(uobj))3578		return PTR_ERR(uobj);3579 3580	obj = container_of(uobj, struct ib_uevent_object, uobject);3581	memset(&resp, 0, sizeof(resp));3582	resp.events_reported = obj->events_reported;3583 3584	uobj_put_destroy(uobj);3585 3586	return uverbs_response(attrs, &resp, sizeof(resp));3587}3588 3589static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)3590{3591	struct ib_uverbs_ex_query_device_resp resp = {};3592	struct ib_uverbs_ex_query_device  cmd;3593	struct ib_device_attr attr = {0};3594	struct ib_ucontext *ucontext;3595	struct ib_device *ib_dev;3596	int err;3597 3598	ucontext = ib_uverbs_get_ucontext(attrs);3599	if (IS_ERR(ucontext))3600		return PTR_ERR(ucontext);3601	ib_dev = ucontext->device;3602 3603	err = uverbs_request(attrs, &cmd, sizeof(cmd));3604	if (err)3605		return err;3606 3607	if (cmd.comp_mask)3608		return -EINVAL;3609 3610	if (cmd.reserved)3611		return -EINVAL;3612 3613	err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata);3614	if (err)3615		return err;3616 3617	copy_query_dev_fields(ucontext, &resp.base, &attr);3618 3619	resp.odp_caps.general_caps = attr.odp_caps.general_caps;3620	resp.odp_caps.per_transport_caps.rc_odp_caps =3621		attr.odp_caps.per_transport_caps.rc_odp_caps;3622	resp.odp_caps.per_transport_caps.uc_odp_caps =3623		attr.odp_caps.per_transport_caps.uc_odp_caps;3624	resp.odp_caps.per_transport_caps.ud_odp_caps =3625		attr.odp_caps.per_transport_caps.ud_odp_caps;3626	resp.xrc_odp_caps = attr.odp_caps.per_transport_caps.xrc_odp_caps;3627 3628	resp.timestamp_mask = attr.timestamp_mask;3629	resp.hca_core_clock = attr.hca_core_clock;3630	resp.device_cap_flags_ex = attr.device_cap_flags;3631	resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;3632	resp.rss_caps.max_rwq_indirection_tables =3633		attr.rss_caps.max_rwq_indirection_tables;3634	resp.rss_caps.max_rwq_indirection_table_size =3635		attr.rss_caps.max_rwq_indirection_table_size;3636	resp.max_wq_type_rq = attr.max_wq_type_rq;3637	resp.raw_packet_caps = attr.raw_packet_caps;3638	resp.tm_caps.max_rndv_hdr_size	= attr.tm_caps.max_rndv_hdr_size;3639	resp.tm_caps.max_num_tags	= attr.tm_caps.max_num_tags;3640	resp.tm_caps.max_ops		= attr.tm_caps.max_ops;3641	resp.tm_caps.max_sge		= attr.tm_caps.max_sge;3642	resp.tm_caps.flags		= attr.tm_caps.flags;3643	resp.cq_moderation_caps.max_cq_moderation_count  =3644		attr.cq_caps.max_cq_moderation_count;3645	resp.cq_moderation_caps.max_cq_moderation_period =3646		attr.cq_caps.max_cq_moderation_period;3647	resp.max_dm_size = attr.max_dm_size;3648	resp.response_length = uverbs_response_length(attrs, sizeof(resp));3649 3650	return uverbs_response(attrs, &resp, sizeof(resp));3651}3652 3653static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)3654{3655	struct ib_uverbs_ex_modify_cq cmd;3656	struct ib_cq *cq;3657	int ret;3658 3659	ret = uverbs_request(attrs, &cmd, sizeof(cmd));3660	if (ret)3661		return ret;3662 3663	if (!cmd.attr_mask || cmd.reserved)3664		return -EINVAL;3665 3666	if (cmd.attr_mask > IB_CQ_MODERATE)3667		return -EOPNOTSUPP;3668 3669	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);3670	if (!cq)3671		return -EINVAL;3672 3673	ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period);3674 3675	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,3676				UVERBS_LOOKUP_READ);3677	return ret;3678}3679 3680/*3681 * Describe the input structs for write(). Some write methods have an input3682 * only struct, most have an input and output. If the struct has an output then3683 * the 'response' u64 must be the first field in the request structure.3684 *3685 * If udata is present then both the request and response structs have a3686 * trailing driver_data flex array. In this case the size of the base struct3687 * cannot be changed.3688 */3689#define UAPI_DEF_WRITE_IO(req, resp)                                           \3690	.write.has_resp = 1 +                                                  \3691			  BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) +    \3692			  BUILD_BUG_ON_ZERO(sizeof_field(req, response) !=    \3693					    sizeof(u64)),                      \3694	.write.req_size = sizeof(req), .write.resp_size = sizeof(resp)3695 3696#define UAPI_DEF_WRITE_I(req) .write.req_size = sizeof(req)3697 3698#define UAPI_DEF_WRITE_UDATA_IO(req, resp)                                     \3699	UAPI_DEF_WRITE_IO(req, resp),                                          \3700		.write.has_udata =                                             \3701			1 +                                                    \3702			BUILD_BUG_ON_ZERO(offsetof(req, driver_data) !=        \3703					  sizeof(req)) +                       \3704			BUILD_BUG_ON_ZERO(offsetof(resp, driver_data) !=       \3705					  sizeof(resp))3706 3707#define UAPI_DEF_WRITE_UDATA_I(req)                                            \3708	UAPI_DEF_WRITE_I(req),                                                 \3709		.write.has_udata =                                             \3710			1 + BUILD_BUG_ON_ZERO(offsetof(req, driver_data) !=    \3711					      sizeof(req))3712 3713/*3714 * The _EX versions are for use with WRITE_EX and allow the last struct member3715 * to be specified. Buffers that do not include that member will be rejected.3716 */3717#define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member)     \3718	.write.has_resp = 1,                                                   \3719	.write.req_size = offsetofend(req, req_last_member),                   \3720	.write.resp_size = offsetofend(resp, resp_last_member)3721 3722#define UAPI_DEF_WRITE_I_EX(req, req_last_member)                              \3723	.write.req_size = offsetofend(req, req_last_member)3724 3725const struct uapi_definition uverbs_def_write_intf[] = {3726	DECLARE_UVERBS_OBJECT(3727		UVERBS_OBJECT_AH,3728		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_AH,3729				     ib_uverbs_create_ah,3730				     UAPI_DEF_WRITE_UDATA_IO(3731					     struct ib_uverbs_create_ah,3732					     struct ib_uverbs_create_ah_resp)),3733		DECLARE_UVERBS_WRITE(3734			IB_USER_VERBS_CMD_DESTROY_AH,3735			ib_uverbs_destroy_ah,3736			UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah)),3737		UAPI_DEF_OBJ_NEEDS_FN(create_user_ah),3738		UAPI_DEF_OBJ_NEEDS_FN(destroy_ah)),3739 3740	DECLARE_UVERBS_OBJECT(3741		UVERBS_OBJECT_COMP_CHANNEL,3742		DECLARE_UVERBS_WRITE(3743			IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL,3744			ib_uverbs_create_comp_channel,3745			UAPI_DEF_WRITE_IO(3746				struct ib_uverbs_create_comp_channel,3747				struct ib_uverbs_create_comp_channel_resp))),3748 3749	DECLARE_UVERBS_OBJECT(3750		UVERBS_OBJECT_CQ,3751		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_CQ,3752				     ib_uverbs_create_cq,3753				     UAPI_DEF_WRITE_UDATA_IO(3754					     struct ib_uverbs_create_cq,3755					     struct ib_uverbs_create_cq_resp),3756				     UAPI_DEF_METHOD_NEEDS_FN(create_cq)),3757		DECLARE_UVERBS_WRITE(3758			IB_USER_VERBS_CMD_DESTROY_CQ,3759			ib_uverbs_destroy_cq,3760			UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_cq,3761					  struct ib_uverbs_destroy_cq_resp),3762			UAPI_DEF_METHOD_NEEDS_FN(destroy_cq)),3763		DECLARE_UVERBS_WRITE(3764			IB_USER_VERBS_CMD_POLL_CQ,3765			ib_uverbs_poll_cq,3766			UAPI_DEF_WRITE_IO(struct ib_uverbs_poll_cq,3767					  struct ib_uverbs_poll_cq_resp),3768			UAPI_DEF_METHOD_NEEDS_FN(poll_cq)),3769		DECLARE_UVERBS_WRITE(3770			IB_USER_VERBS_CMD_REQ_NOTIFY_CQ,3771			ib_uverbs_req_notify_cq,3772			UAPI_DEF_WRITE_I(struct ib_uverbs_req_notify_cq),3773			UAPI_DEF_METHOD_NEEDS_FN(req_notify_cq)),3774		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_RESIZE_CQ,3775				     ib_uverbs_resize_cq,3776				     UAPI_DEF_WRITE_UDATA_IO(3777					     struct ib_uverbs_resize_cq,3778					     struct ib_uverbs_resize_cq_resp),3779				     UAPI_DEF_METHOD_NEEDS_FN(resize_cq)),3780		DECLARE_UVERBS_WRITE_EX(3781			IB_USER_VERBS_EX_CMD_CREATE_CQ,3782			ib_uverbs_ex_create_cq,3783			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_cq,3784					     reserved,3785					     struct ib_uverbs_ex_create_cq_resp,3786					     response_length),3787			UAPI_DEF_METHOD_NEEDS_FN(create_cq)),3788		DECLARE_UVERBS_WRITE_EX(3789			IB_USER_VERBS_EX_CMD_MODIFY_CQ,3790			ib_uverbs_ex_modify_cq,3791			UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq),3792			UAPI_DEF_METHOD_NEEDS_FN(modify_cq))),3793 3794	DECLARE_UVERBS_OBJECT(3795		UVERBS_OBJECT_DEVICE,3796		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_GET_CONTEXT,3797				     ib_uverbs_get_context,3798				     UAPI_DEF_WRITE_UDATA_IO(3799					     struct ib_uverbs_get_context,3800					     struct ib_uverbs_get_context_resp)),3801		DECLARE_UVERBS_WRITE(3802			IB_USER_VERBS_CMD_QUERY_DEVICE,3803			ib_uverbs_query_device,3804			UAPI_DEF_WRITE_IO(struct ib_uverbs_query_device,3805					  struct ib_uverbs_query_device_resp)),3806		DECLARE_UVERBS_WRITE(3807			IB_USER_VERBS_CMD_QUERY_PORT,3808			ib_uverbs_query_port,3809			UAPI_DEF_WRITE_IO(struct ib_uverbs_query_port,3810					  struct ib_uverbs_query_port_resp),3811			UAPI_DEF_METHOD_NEEDS_FN(query_port)),3812		DECLARE_UVERBS_WRITE_EX(3813			IB_USER_VERBS_EX_CMD_QUERY_DEVICE,3814			ib_uverbs_ex_query_device,3815			UAPI_DEF_WRITE_IO_EX(3816				struct ib_uverbs_ex_query_device,3817				reserved,3818				struct ib_uverbs_ex_query_device_resp,3819				response_length),3820			UAPI_DEF_METHOD_NEEDS_FN(query_device)),3821		UAPI_DEF_OBJ_NEEDS_FN(alloc_ucontext),3822		UAPI_DEF_OBJ_NEEDS_FN(dealloc_ucontext)),3823 3824	DECLARE_UVERBS_OBJECT(3825		UVERBS_OBJECT_FLOW,3826		DECLARE_UVERBS_WRITE_EX(3827			IB_USER_VERBS_EX_CMD_CREATE_FLOW,3828			ib_uverbs_ex_create_flow,3829			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_create_flow,3830					     flow_attr,3831					     struct ib_uverbs_create_flow_resp,3832					     flow_handle),3833			UAPI_DEF_METHOD_NEEDS_FN(create_flow)),3834		DECLARE_UVERBS_WRITE_EX(3835			IB_USER_VERBS_EX_CMD_DESTROY_FLOW,3836			ib_uverbs_ex_destroy_flow,3837			UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_flow),3838			UAPI_DEF_METHOD_NEEDS_FN(destroy_flow))),3839 3840	DECLARE_UVERBS_OBJECT(3841		UVERBS_OBJECT_MR,3842		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_DEREG_MR,3843				     ib_uverbs_dereg_mr,3844				     UAPI_DEF_WRITE_I(struct ib_uverbs_dereg_mr),3845				     UAPI_DEF_METHOD_NEEDS_FN(dereg_mr)),3846		DECLARE_UVERBS_WRITE(3847			IB_USER_VERBS_CMD_REG_MR,3848			ib_uverbs_reg_mr,3849			UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_reg_mr,3850						struct ib_uverbs_reg_mr_resp),3851			UAPI_DEF_METHOD_NEEDS_FN(reg_user_mr)),3852		DECLARE_UVERBS_WRITE(3853			IB_USER_VERBS_CMD_REREG_MR,3854			ib_uverbs_rereg_mr,3855			UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_rereg_mr,3856						struct ib_uverbs_rereg_mr_resp),3857			UAPI_DEF_METHOD_NEEDS_FN(rereg_user_mr))),3858 3859	DECLARE_UVERBS_OBJECT(3860		UVERBS_OBJECT_MW,3861		DECLARE_UVERBS_WRITE(3862			IB_USER_VERBS_CMD_ALLOC_MW,3863			ib_uverbs_alloc_mw,3864			UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_mw,3865						struct ib_uverbs_alloc_mw_resp),3866			UAPI_DEF_METHOD_NEEDS_FN(alloc_mw)),3867		DECLARE_UVERBS_WRITE(3868			IB_USER_VERBS_CMD_DEALLOC_MW,3869			ib_uverbs_dealloc_mw,3870			UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_mw),3871			UAPI_DEF_METHOD_NEEDS_FN(dealloc_mw))),3872 3873	DECLARE_UVERBS_OBJECT(3874		UVERBS_OBJECT_PD,3875		DECLARE_UVERBS_WRITE(3876			IB_USER_VERBS_CMD_ALLOC_PD,3877			ib_uverbs_alloc_pd,3878			UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_pd,3879						struct ib_uverbs_alloc_pd_resp),3880			UAPI_DEF_METHOD_NEEDS_FN(alloc_pd)),3881		DECLARE_UVERBS_WRITE(3882			IB_USER_VERBS_CMD_DEALLOC_PD,3883			ib_uverbs_dealloc_pd,3884			UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_pd),3885			UAPI_DEF_METHOD_NEEDS_FN(dealloc_pd))),3886 3887	DECLARE_UVERBS_OBJECT(3888		UVERBS_OBJECT_QP,3889		DECLARE_UVERBS_WRITE(3890			IB_USER_VERBS_CMD_ATTACH_MCAST,3891			ib_uverbs_attach_mcast,3892			UAPI_DEF_WRITE_I(struct ib_uverbs_attach_mcast),3893			UAPI_DEF_METHOD_NEEDS_FN(attach_mcast),3894			UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),3895		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_QP,3896				     ib_uverbs_create_qp,3897				     UAPI_DEF_WRITE_UDATA_IO(3898					     struct ib_uverbs_create_qp,3899					     struct ib_uverbs_create_qp_resp),3900				     UAPI_DEF_METHOD_NEEDS_FN(create_qp)),3901		DECLARE_UVERBS_WRITE(3902			IB_USER_VERBS_CMD_DESTROY_QP,3903			ib_uverbs_destroy_qp,3904			UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_qp,3905					  struct ib_uverbs_destroy_qp_resp),3906			UAPI_DEF_METHOD_NEEDS_FN(destroy_qp)),3907		DECLARE_UVERBS_WRITE(3908			IB_USER_VERBS_CMD_DETACH_MCAST,3909			ib_uverbs_detach_mcast,3910			UAPI_DEF_WRITE_I(struct ib_uverbs_detach_mcast),3911			UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),3912		DECLARE_UVERBS_WRITE(3913			IB_USER_VERBS_CMD_MODIFY_QP,3914			ib_uverbs_modify_qp,3915			UAPI_DEF_WRITE_I(struct ib_uverbs_modify_qp),3916			UAPI_DEF_METHOD_NEEDS_FN(modify_qp)),3917		DECLARE_UVERBS_WRITE(3918			IB_USER_VERBS_CMD_POST_RECV,3919			ib_uverbs_post_recv,3920			UAPI_DEF_WRITE_IO(struct ib_uverbs_post_recv,3921					  struct ib_uverbs_post_recv_resp),3922			UAPI_DEF_METHOD_NEEDS_FN(post_recv)),3923		DECLARE_UVERBS_WRITE(3924			IB_USER_VERBS_CMD_POST_SEND,3925			ib_uverbs_post_send,3926			UAPI_DEF_WRITE_IO(struct ib_uverbs_post_send,3927					  struct ib_uverbs_post_send_resp),3928			UAPI_DEF_METHOD_NEEDS_FN(post_send)),3929		DECLARE_UVERBS_WRITE(3930			IB_USER_VERBS_CMD_QUERY_QP,3931			ib_uverbs_query_qp,3932			UAPI_DEF_WRITE_IO(struct ib_uverbs_query_qp,3933					  struct ib_uverbs_query_qp_resp),3934			UAPI_DEF_METHOD_NEEDS_FN(query_qp)),3935		DECLARE_UVERBS_WRITE_EX(3936			IB_USER_VERBS_EX_CMD_CREATE_QP,3937			ib_uverbs_ex_create_qp,3938			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_qp,3939					     comp_mask,3940					     struct ib_uverbs_ex_create_qp_resp,3941					     response_length),3942			UAPI_DEF_METHOD_NEEDS_FN(create_qp)),3943		DECLARE_UVERBS_WRITE_EX(3944			IB_USER_VERBS_EX_CMD_MODIFY_QP,3945			ib_uverbs_ex_modify_qp,3946			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_modify_qp,3947					     base,3948					     struct ib_uverbs_ex_modify_qp_resp,3949					     response_length),3950			UAPI_DEF_METHOD_NEEDS_FN(modify_qp))),3951 3952	DECLARE_UVERBS_OBJECT(3953		UVERBS_OBJECT_RWQ_IND_TBL,3954		DECLARE_UVERBS_WRITE_EX(3955			IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL,3956			ib_uverbs_ex_create_rwq_ind_table,3957			UAPI_DEF_WRITE_IO_EX(3958				struct ib_uverbs_ex_create_rwq_ind_table,3959				log_ind_tbl_size,3960				struct ib_uverbs_ex_create_rwq_ind_table_resp,3961				ind_tbl_num),3962			UAPI_DEF_METHOD_NEEDS_FN(create_rwq_ind_table)),3963		DECLARE_UVERBS_WRITE_EX(3964			IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL,3965			ib_uverbs_ex_destroy_rwq_ind_table,3966			UAPI_DEF_WRITE_I(3967				struct ib_uverbs_ex_destroy_rwq_ind_table),3968			UAPI_DEF_METHOD_NEEDS_FN(destroy_rwq_ind_table))),3969 3970	DECLARE_UVERBS_OBJECT(3971		UVERBS_OBJECT_WQ,3972		DECLARE_UVERBS_WRITE_EX(3973			IB_USER_VERBS_EX_CMD_CREATE_WQ,3974			ib_uverbs_ex_create_wq,3975			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_wq,3976					     max_sge,3977					     struct ib_uverbs_ex_create_wq_resp,3978					     wqn),3979			UAPI_DEF_METHOD_NEEDS_FN(create_wq)),3980		DECLARE_UVERBS_WRITE_EX(3981			IB_USER_VERBS_EX_CMD_DESTROY_WQ,3982			ib_uverbs_ex_destroy_wq,3983			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_destroy_wq,3984					     wq_handle,3985					     struct ib_uverbs_ex_destroy_wq_resp,3986					     reserved),3987			UAPI_DEF_METHOD_NEEDS_FN(destroy_wq)),3988		DECLARE_UVERBS_WRITE_EX(3989			IB_USER_VERBS_EX_CMD_MODIFY_WQ,3990			ib_uverbs_ex_modify_wq,3991			UAPI_DEF_WRITE_I_EX(struct ib_uverbs_ex_modify_wq,3992					    curr_wq_state),3993			UAPI_DEF_METHOD_NEEDS_FN(modify_wq))),3994 3995	DECLARE_UVERBS_OBJECT(3996		UVERBS_OBJECT_SRQ,3997		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_SRQ,3998				     ib_uverbs_create_srq,3999				     UAPI_DEF_WRITE_UDATA_IO(4000					     struct ib_uverbs_create_srq,4001					     struct ib_uverbs_create_srq_resp),4002				     UAPI_DEF_METHOD_NEEDS_FN(create_srq)),4003		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_XSRQ,4004				     ib_uverbs_create_xsrq,4005				     UAPI_DEF_WRITE_UDATA_IO(4006					     struct ib_uverbs_create_xsrq,4007					     struct ib_uverbs_create_srq_resp),4008				     UAPI_DEF_METHOD_NEEDS_FN(create_srq)),4009		DECLARE_UVERBS_WRITE(4010			IB_USER_VERBS_CMD_DESTROY_SRQ,4011			ib_uverbs_destroy_srq,4012			UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_srq,4013					  struct ib_uverbs_destroy_srq_resp),4014			UAPI_DEF_METHOD_NEEDS_FN(destroy_srq)),4015		DECLARE_UVERBS_WRITE(4016			IB_USER_VERBS_CMD_MODIFY_SRQ,4017			ib_uverbs_modify_srq,4018			UAPI_DEF_WRITE_UDATA_I(struct ib_uverbs_modify_srq),4019			UAPI_DEF_METHOD_NEEDS_FN(modify_srq)),4020		DECLARE_UVERBS_WRITE(4021			IB_USER_VERBS_CMD_POST_SRQ_RECV,4022			ib_uverbs_post_srq_recv,4023			UAPI_DEF_WRITE_IO(struct ib_uverbs_post_srq_recv,4024					  struct ib_uverbs_post_srq_recv_resp),4025			UAPI_DEF_METHOD_NEEDS_FN(post_srq_recv)),4026		DECLARE_UVERBS_WRITE(4027			IB_USER_VERBS_CMD_QUERY_SRQ,4028			ib_uverbs_query_srq,4029			UAPI_DEF_WRITE_IO(struct ib_uverbs_query_srq,4030					  struct ib_uverbs_query_srq_resp),4031			UAPI_DEF_METHOD_NEEDS_FN(query_srq))),4032 4033	DECLARE_UVERBS_OBJECT(4034		UVERBS_OBJECT_XRCD,4035		DECLARE_UVERBS_WRITE(4036			IB_USER_VERBS_CMD_CLOSE_XRCD,4037			ib_uverbs_close_xrcd,4038			UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd)),4039		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP,4040				     ib_uverbs_open_qp,4041				     UAPI_DEF_WRITE_UDATA_IO(4042					     struct ib_uverbs_open_qp,4043					     struct ib_uverbs_create_qp_resp)),4044		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_XRCD,4045				     ib_uverbs_open_xrcd,4046				     UAPI_DEF_WRITE_UDATA_IO(4047					     struct ib_uverbs_open_xrcd,4048					     struct ib_uverbs_open_xrcd_resp)),4049		UAPI_DEF_OBJ_NEEDS_FN(alloc_xrcd),4050		UAPI_DEF_OBJ_NEEDS_FN(dealloc_xrcd)),4051 4052	{},4053};4054