brintos / linux-shallow public Read only

1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */2/*3 * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.4 * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.5 * Copyright (c) 2004, 2020 Intel Corporation.  All rights reserved.6 * Copyright (c) 2004 Topspin Corporation.  All rights reserved.7 * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.8 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.9 * Copyright (c) 2005, 2006, 2007 Cisco Systems.  All rights reserved.10 */11 12#ifndef IB_VERBS_H13#define IB_VERBS_H14 15#include <linux/ethtool.h>16#include <linux/types.h>17#include <linux/device.h>18#include <linux/dma-mapping.h>19#include <linux/kref.h>20#include <linux/list.h>21#include <linux/rwsem.h>22#include <linux/workqueue.h>23#include <linux/irq_poll.h>24#include <uapi/linux/if_ether.h>25#include <net/ipv6.h>26#include <net/ip.h>27#include <linux/string.h>28#include <linux/slab.h>29#include <linux/netdevice.h>30#include <linux/refcount.h>31#include <linux/if_link.h>32#include <linux/atomic.h>33#include <linux/mmu_notifier.h>34#include <linux/uaccess.h>35#include <linux/cgroup_rdma.h>36#include <linux/irqflags.h>37#include <linux/preempt.h>38#include <linux/dim.h>39#include <uapi/rdma/ib_user_verbs.h>40#include <rdma/rdma_counter.h>41#include <rdma/restrack.h>42#include <rdma/signature.h>43#include <uapi/rdma/rdma_user_ioctl.h>44#include <uapi/rdma/ib_user_ioctl_verbs.h>45 46#define IB_FW_VERSION_NAME_MAX	ETHTOOL_FWVERS_LEN47 48struct ib_umem_odp;49struct ib_uqp_object;50struct ib_usrq_object;51struct ib_uwq_object;52struct rdma_cm_id;53struct ib_port;54struct hw_stats_device_data;55 56extern struct workqueue_struct *ib_wq;57extern struct workqueue_struct *ib_comp_wq;58extern struct workqueue_struct *ib_comp_unbound_wq;59 60struct ib_ucq_object;61 62__printf(3, 4) __cold63void ibdev_printk(const char *level, const struct ib_device *ibdev,64		  const char *format, ...);65__printf(2, 3) __cold66void ibdev_emerg(const struct ib_device *ibdev, const char *format, ...);67__printf(2, 3) __cold68void ibdev_alert(const struct ib_device *ibdev, const char *format, ...);69__printf(2, 3) __cold70void ibdev_crit(const struct ib_device *ibdev, const char *format, ...);71__printf(2, 3) __cold72void ibdev_err(const struct ib_device *ibdev, const char *format, ...);73__printf(2, 3) __cold74void ibdev_warn(const struct ib_device *ibdev, const char *format, ...);75__printf(2, 3) __cold76void ibdev_notice(const struct ib_device *ibdev, const char *format, ...);77__printf(2, 3) __cold78void ibdev_info(const struct ib_device *ibdev, const char *format, ...);79 80#if defined(CONFIG_DYNAMIC_DEBUG) || \81	(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))82#define ibdev_dbg(__dev, format, args...)                       \83	dynamic_ibdev_dbg(__dev, format, ##args)84#else85__printf(2, 3) __cold86static inline87void ibdev_dbg(const struct ib_device *ibdev, const char *format, ...) {}88#endif89 90#define ibdev_level_ratelimited(ibdev_level, ibdev, fmt, ...)           \91do {                                                                    \92	static DEFINE_RATELIMIT_STATE(_rs,                              \93				      DEFAULT_RATELIMIT_INTERVAL,       \94				      DEFAULT_RATELIMIT_BURST);         \95	if (__ratelimit(&_rs))                                          \96		ibdev_level(ibdev, fmt, ##__VA_ARGS__);                 \97} while (0)98 99#define ibdev_emerg_ratelimited(ibdev, fmt, ...) \100	ibdev_level_ratelimited(ibdev_emerg, ibdev, fmt, ##__VA_ARGS__)101#define ibdev_alert_ratelimited(ibdev, fmt, ...) \102	ibdev_level_ratelimited(ibdev_alert, ibdev, fmt, ##__VA_ARGS__)103#define ibdev_crit_ratelimited(ibdev, fmt, ...) \104	ibdev_level_ratelimited(ibdev_crit, ibdev, fmt, ##__VA_ARGS__)105#define ibdev_err_ratelimited(ibdev, fmt, ...) \106	ibdev_level_ratelimited(ibdev_err, ibdev, fmt, ##__VA_ARGS__)107#define ibdev_warn_ratelimited(ibdev, fmt, ...) \108	ibdev_level_ratelimited(ibdev_warn, ibdev, fmt, ##__VA_ARGS__)109#define ibdev_notice_ratelimited(ibdev, fmt, ...) \110	ibdev_level_ratelimited(ibdev_notice, ibdev, fmt, ##__VA_ARGS__)111#define ibdev_info_ratelimited(ibdev, fmt, ...) \112	ibdev_level_ratelimited(ibdev_info, ibdev, fmt, ##__VA_ARGS__)113 114#if defined(CONFIG_DYNAMIC_DEBUG) || \115	(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))116/* descriptor check is first to prevent flooding with "callbacks suppressed" */117#define ibdev_dbg_ratelimited(ibdev, fmt, ...)                          \118do {                                                                    \119	static DEFINE_RATELIMIT_STATE(_rs,                              \120				      DEFAULT_RATELIMIT_INTERVAL,       \121				      DEFAULT_RATELIMIT_BURST);         \122	DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, fmt);                 \123	if (DYNAMIC_DEBUG_BRANCH(descriptor) && __ratelimit(&_rs))      \124		__dynamic_ibdev_dbg(&descriptor, ibdev, fmt,            \125				    ##__VA_ARGS__);                     \126} while (0)127#else128__printf(2, 3) __cold129static inline130void ibdev_dbg_ratelimited(const struct ib_device *ibdev, const char *format, ...) {}131#endif132 133union ib_gid {134	u8	raw[16];135	struct {136		__be64	subnet_prefix;137		__be64	interface_id;138	} global;139};140 141extern union ib_gid zgid;142 143enum ib_gid_type {144	IB_GID_TYPE_IB = IB_UVERBS_GID_TYPE_IB,145	IB_GID_TYPE_ROCE = IB_UVERBS_GID_TYPE_ROCE_V1,146	IB_GID_TYPE_ROCE_UDP_ENCAP = IB_UVERBS_GID_TYPE_ROCE_V2,147	IB_GID_TYPE_SIZE148};149 150#define ROCE_V2_UDP_DPORT      4791151struct ib_gid_attr {152	struct net_device __rcu	*ndev;153	struct ib_device	*device;154	union ib_gid		gid;155	enum ib_gid_type	gid_type;156	u16			index;157	u32			port_num;158};159 160enum {161	/* set the local administered indication */162	IB_SA_WELL_KNOWN_GUID	= BIT_ULL(57) | 2,163};164 165enum rdma_transport_type {166	RDMA_TRANSPORT_IB,167	RDMA_TRANSPORT_IWARP,168	RDMA_TRANSPORT_USNIC,169	RDMA_TRANSPORT_USNIC_UDP,170	RDMA_TRANSPORT_UNSPECIFIED,171};172 173enum rdma_protocol_type {174	RDMA_PROTOCOL_IB,175	RDMA_PROTOCOL_IBOE,176	RDMA_PROTOCOL_IWARP,177	RDMA_PROTOCOL_USNIC_UDP178};179 180__attribute_const__ enum rdma_transport_type181rdma_node_get_transport(unsigned int node_type);182 183enum rdma_network_type {184	RDMA_NETWORK_IB,185	RDMA_NETWORK_ROCE_V1,186	RDMA_NETWORK_IPV4,187	RDMA_NETWORK_IPV6188};189 190static inline enum ib_gid_type ib_network_to_gid_type(enum rdma_network_type network_type)191{192	if (network_type == RDMA_NETWORK_IPV4 ||193	    network_type == RDMA_NETWORK_IPV6)194		return IB_GID_TYPE_ROCE_UDP_ENCAP;195	else if (network_type == RDMA_NETWORK_ROCE_V1)196		return IB_GID_TYPE_ROCE;197	else198		return IB_GID_TYPE_IB;199}200 201static inline enum rdma_network_type202rdma_gid_attr_network_type(const struct ib_gid_attr *attr)203{204	if (attr->gid_type == IB_GID_TYPE_IB)205		return RDMA_NETWORK_IB;206 207	if (attr->gid_type == IB_GID_TYPE_ROCE)208		return RDMA_NETWORK_ROCE_V1;209 210	if (ipv6_addr_v4mapped((struct in6_addr *)&attr->gid))211		return RDMA_NETWORK_IPV4;212	else213		return RDMA_NETWORK_IPV6;214}215 216enum rdma_link_layer {217	IB_LINK_LAYER_UNSPECIFIED,218	IB_LINK_LAYER_INFINIBAND,219	IB_LINK_LAYER_ETHERNET,220};221 222enum ib_device_cap_flags {223	IB_DEVICE_RESIZE_MAX_WR = IB_UVERBS_DEVICE_RESIZE_MAX_WR,224	IB_DEVICE_BAD_PKEY_CNTR = IB_UVERBS_DEVICE_BAD_PKEY_CNTR,225	IB_DEVICE_BAD_QKEY_CNTR = IB_UVERBS_DEVICE_BAD_QKEY_CNTR,226	IB_DEVICE_RAW_MULTI = IB_UVERBS_DEVICE_RAW_MULTI,227	IB_DEVICE_AUTO_PATH_MIG = IB_UVERBS_DEVICE_AUTO_PATH_MIG,228	IB_DEVICE_CHANGE_PHY_PORT = IB_UVERBS_DEVICE_CHANGE_PHY_PORT,229	IB_DEVICE_UD_AV_PORT_ENFORCE = IB_UVERBS_DEVICE_UD_AV_PORT_ENFORCE,230	IB_DEVICE_CURR_QP_STATE_MOD = IB_UVERBS_DEVICE_CURR_QP_STATE_MOD,231	IB_DEVICE_SHUTDOWN_PORT = IB_UVERBS_DEVICE_SHUTDOWN_PORT,232	/* IB_DEVICE_INIT_TYPE = IB_UVERBS_DEVICE_INIT_TYPE, (not in use) */233	IB_DEVICE_PORT_ACTIVE_EVENT = IB_UVERBS_DEVICE_PORT_ACTIVE_EVENT,234	IB_DEVICE_SYS_IMAGE_GUID = IB_UVERBS_DEVICE_SYS_IMAGE_GUID,235	IB_DEVICE_RC_RNR_NAK_GEN = IB_UVERBS_DEVICE_RC_RNR_NAK_GEN,236	IB_DEVICE_SRQ_RESIZE = IB_UVERBS_DEVICE_SRQ_RESIZE,237	IB_DEVICE_N_NOTIFY_CQ = IB_UVERBS_DEVICE_N_NOTIFY_CQ,238 239	/* Reserved, old SEND_W_INV = 1 << 16,*/240	IB_DEVICE_MEM_WINDOW = IB_UVERBS_DEVICE_MEM_WINDOW,241	/*242	 * Devices should set IB_DEVICE_UD_IP_SUM if they support243	 * insertion of UDP and TCP checksum on outgoing UD IPoIB244	 * messages and can verify the validity of checksum for245	 * incoming messages.  Setting this flag implies that the246	 * IPoIB driver may set NETIF_F_IP_CSUM for datagram mode.247	 */248	IB_DEVICE_UD_IP_CSUM = IB_UVERBS_DEVICE_UD_IP_CSUM,249	IB_DEVICE_XRC = IB_UVERBS_DEVICE_XRC,250 251	/*252	 * This device supports the IB "base memory management extension",253	 * which includes support for fast registrations (IB_WR_REG_MR,254	 * IB_WR_LOCAL_INV and IB_WR_SEND_WITH_INV verbs).  This flag should255	 * also be set by any iWarp device which must support FRs to comply256	 * to the iWarp verbs spec.  iWarp devices also support the257	 * IB_WR_RDMA_READ_WITH_INV verb for RDMA READs that invalidate the258	 * stag.259	 */260	IB_DEVICE_MEM_MGT_EXTENSIONS = IB_UVERBS_DEVICE_MEM_MGT_EXTENSIONS,261	IB_DEVICE_MEM_WINDOW_TYPE_2A = IB_UVERBS_DEVICE_MEM_WINDOW_TYPE_2A,262	IB_DEVICE_MEM_WINDOW_TYPE_2B = IB_UVERBS_DEVICE_MEM_WINDOW_TYPE_2B,263	IB_DEVICE_RC_IP_CSUM = IB_UVERBS_DEVICE_RC_IP_CSUM,264	/* Deprecated. Please use IB_RAW_PACKET_CAP_IP_CSUM. */265	IB_DEVICE_RAW_IP_CSUM = IB_UVERBS_DEVICE_RAW_IP_CSUM,266	IB_DEVICE_MANAGED_FLOW_STEERING =267		IB_UVERBS_DEVICE_MANAGED_FLOW_STEERING,268	/* Deprecated. Please use IB_RAW_PACKET_CAP_SCATTER_FCS. */269	IB_DEVICE_RAW_SCATTER_FCS = IB_UVERBS_DEVICE_RAW_SCATTER_FCS,270	/* The device supports padding incoming writes to cacheline. */271	IB_DEVICE_PCI_WRITE_END_PADDING =272		IB_UVERBS_DEVICE_PCI_WRITE_END_PADDING,273	/* Placement type attributes */274	IB_DEVICE_FLUSH_GLOBAL = IB_UVERBS_DEVICE_FLUSH_GLOBAL,275	IB_DEVICE_FLUSH_PERSISTENT = IB_UVERBS_DEVICE_FLUSH_PERSISTENT,276	IB_DEVICE_ATOMIC_WRITE = IB_UVERBS_DEVICE_ATOMIC_WRITE,277};278 279enum ib_kernel_cap_flags {280	/*281	 * This device supports a per-device lkey or stag that can be282	 * used without performing a memory registration for the local283	 * memory.  Note that ULPs should never check this flag, but284	 * instead of use the local_dma_lkey flag in the ib_pd structure,285	 * which will always contain a usable lkey.286	 */287	IBK_LOCAL_DMA_LKEY = 1 << 0,288	/* IB_QP_CREATE_INTEGRITY_EN is supported to implement T10-PI */289	IBK_INTEGRITY_HANDOVER = 1 << 1,290	/* IB_ACCESS_ON_DEMAND is supported during reg_user_mr() */291	IBK_ON_DEMAND_PAGING = 1 << 2,292	/* IB_MR_TYPE_SG_GAPS is supported */293	IBK_SG_GAPS_REG = 1 << 3,294	/* Driver supports RDMA_NLDEV_CMD_DELLINK */295	IBK_ALLOW_USER_UNREG = 1 << 4,296 297	/* ipoib will use IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK */298	IBK_BLOCK_MULTICAST_LOOPBACK = 1 << 5,299	/* iopib will use IB_QP_CREATE_IPOIB_UD_LSO for its QPs */300	IBK_UD_TSO = 1 << 6,301	/* iopib will use the device ops:302	 *   get_vf_config303	 *   get_vf_guid304	 *   get_vf_stats305	 *   set_vf_guid306	 *   set_vf_link_state307	 */308	IBK_VIRTUAL_FUNCTION = 1 << 7,309	/* ipoib will use IB_QP_CREATE_NETDEV_USE for its QPs */310	IBK_RDMA_NETDEV_OPA = 1 << 8,311};312 313enum ib_atomic_cap {314	IB_ATOMIC_NONE,315	IB_ATOMIC_HCA,316	IB_ATOMIC_GLOB317};318 319enum ib_odp_general_cap_bits {320	IB_ODP_SUPPORT		= 1 << 0,321	IB_ODP_SUPPORT_IMPLICIT = 1 << 1,322};323 324enum ib_odp_transport_cap_bits {325	IB_ODP_SUPPORT_SEND	= 1 << 0,326	IB_ODP_SUPPORT_RECV	= 1 << 1,327	IB_ODP_SUPPORT_WRITE	= 1 << 2,328	IB_ODP_SUPPORT_READ	= 1 << 3,329	IB_ODP_SUPPORT_ATOMIC	= 1 << 4,330	IB_ODP_SUPPORT_SRQ_RECV	= 1 << 5,331};332 333struct ib_odp_caps {334	uint64_t general_caps;335	struct {336		uint32_t  rc_odp_caps;337		uint32_t  uc_odp_caps;338		uint32_t  ud_odp_caps;339		uint32_t  xrc_odp_caps;340	} per_transport_caps;341};342 343struct ib_rss_caps {344	/* Corresponding bit will be set if qp type from345	 * 'enum ib_qp_type' is supported, e.g.346	 * supported_qpts |= 1 << IB_QPT_UD347	 */348	u32 supported_qpts;349	u32 max_rwq_indirection_tables;350	u32 max_rwq_indirection_table_size;351};352 353enum ib_tm_cap_flags {354	/*  Support tag matching with rendezvous offload for RC transport */355	IB_TM_CAP_RNDV_RC = 1 << 0,356};357 358struct ib_tm_caps {359	/* Max size of RNDV header */360	u32 max_rndv_hdr_size;361	/* Max number of entries in tag matching list */362	u32 max_num_tags;363	/* From enum ib_tm_cap_flags */364	u32 flags;365	/* Max number of outstanding list operations */366	u32 max_ops;367	/* Max number of SGE in tag matching entry */368	u32 max_sge;369};370 371struct ib_cq_init_attr {372	unsigned int	cqe;373	u32		comp_vector;374	u32		flags;375};376 377enum ib_cq_attr_mask {378	IB_CQ_MODERATE = 1 << 0,379};380 381struct ib_cq_caps {382	u16     max_cq_moderation_count;383	u16     max_cq_moderation_period;384};385 386struct ib_dm_mr_attr {387	u64		length;388	u64		offset;389	u32		access_flags;390};391 392struct ib_dm_alloc_attr {393	u64	length;394	u32	alignment;395	u32	flags;396};397 398struct ib_device_attr {399	u64			fw_ver;400	__be64			sys_image_guid;401	u64			max_mr_size;402	u64			page_size_cap;403	u32			vendor_id;404	u32			vendor_part_id;405	u32			hw_ver;406	int			max_qp;407	int			max_qp_wr;408	u64			device_cap_flags;409	u64			kernel_cap_flags;410	int			max_send_sge;411	int			max_recv_sge;412	int			max_sge_rd;413	int			max_cq;414	int			max_cqe;415	int			max_mr;416	int			max_pd;417	int			max_qp_rd_atom;418	int			max_ee_rd_atom;419	int			max_res_rd_atom;420	int			max_qp_init_rd_atom;421	int			max_ee_init_rd_atom;422	enum ib_atomic_cap	atomic_cap;423	enum ib_atomic_cap	masked_atomic_cap;424	int			max_ee;425	int			max_rdd;426	int			max_mw;427	int			max_raw_ipv6_qp;428	int			max_raw_ethy_qp;429	int			max_mcast_grp;430	int			max_mcast_qp_attach;431	int			max_total_mcast_qp_attach;432	int			max_ah;433	int			max_srq;434	int			max_srq_wr;435	int			max_srq_sge;436	unsigned int		max_fast_reg_page_list_len;437	unsigned int		max_pi_fast_reg_page_list_len;438	u16			max_pkeys;439	u8			local_ca_ack_delay;440	int			sig_prot_cap;441	int			sig_guard_cap;442	struct ib_odp_caps	odp_caps;443	uint64_t		timestamp_mask;444	uint64_t		hca_core_clock; /* in KHZ */445	struct ib_rss_caps	rss_caps;446	u32			max_wq_type_rq;447	u32			raw_packet_caps; /* Use ib_raw_packet_caps enum */448	struct ib_tm_caps	tm_caps;449	struct ib_cq_caps       cq_caps;450	u64			max_dm_size;451	/* Max entries for sgl for optimized performance per READ */452	u32			max_sgl_rd;453};454 455enum ib_mtu {456	IB_MTU_256  = 1,457	IB_MTU_512  = 2,458	IB_MTU_1024 = 3,459	IB_MTU_2048 = 4,460	IB_MTU_4096 = 5461};462 463enum opa_mtu {464	OPA_MTU_8192 = 6,465	OPA_MTU_10240 = 7466};467 468static inline int ib_mtu_enum_to_int(enum ib_mtu mtu)469{470	switch (mtu) {471	case IB_MTU_256:  return  256;472	case IB_MTU_512:  return  512;473	case IB_MTU_1024: return 1024;474	case IB_MTU_2048: return 2048;475	case IB_MTU_4096: return 4096;476	default: 	  return -1;477	}478}479 480static inline enum ib_mtu ib_mtu_int_to_enum(int mtu)481{482	if (mtu >= 4096)483		return IB_MTU_4096;484	else if (mtu >= 2048)485		return IB_MTU_2048;486	else if (mtu >= 1024)487		return IB_MTU_1024;488	else if (mtu >= 512)489		return IB_MTU_512;490	else491		return IB_MTU_256;492}493 494static inline int opa_mtu_enum_to_int(enum opa_mtu mtu)495{496	switch (mtu) {497	case OPA_MTU_8192:498		return 8192;499	case OPA_MTU_10240:500		return 10240;501	default:502		return(ib_mtu_enum_to_int((enum ib_mtu)mtu));503	}504}505 506static inline enum opa_mtu opa_mtu_int_to_enum(int mtu)507{508	if (mtu >= 10240)509		return OPA_MTU_10240;510	else if (mtu >= 8192)511		return OPA_MTU_8192;512	else513		return ((enum opa_mtu)ib_mtu_int_to_enum(mtu));514}515 516enum ib_port_state {517	IB_PORT_NOP		= 0,518	IB_PORT_DOWN		= 1,519	IB_PORT_INIT		= 2,520	IB_PORT_ARMED		= 3,521	IB_PORT_ACTIVE		= 4,522	IB_PORT_ACTIVE_DEFER	= 5523};524 525enum ib_port_phys_state {526	IB_PORT_PHYS_STATE_SLEEP = 1,527	IB_PORT_PHYS_STATE_POLLING = 2,528	IB_PORT_PHYS_STATE_DISABLED = 3,529	IB_PORT_PHYS_STATE_PORT_CONFIGURATION_TRAINING = 4,530	IB_PORT_PHYS_STATE_LINK_UP = 5,531	IB_PORT_PHYS_STATE_LINK_ERROR_RECOVERY = 6,532	IB_PORT_PHYS_STATE_PHY_TEST = 7,533};534 535enum ib_port_width {536	IB_WIDTH_1X	= 1,537	IB_WIDTH_2X	= 16,538	IB_WIDTH_4X	= 2,539	IB_WIDTH_8X	= 4,540	IB_WIDTH_12X	= 8541};542 543static inline int ib_width_enum_to_int(enum ib_port_width width)544{545	switch (width) {546	case IB_WIDTH_1X:  return  1;547	case IB_WIDTH_2X:  return  2;548	case IB_WIDTH_4X:  return  4;549	case IB_WIDTH_8X:  return  8;550	case IB_WIDTH_12X: return 12;551	default: 	  return -1;552	}553}554 555enum ib_port_speed {556	IB_SPEED_SDR	= 1,557	IB_SPEED_DDR	= 2,558	IB_SPEED_QDR	= 4,559	IB_SPEED_FDR10	= 8,560	IB_SPEED_FDR	= 16,561	IB_SPEED_EDR	= 32,562	IB_SPEED_HDR	= 64,563	IB_SPEED_NDR	= 128,564	IB_SPEED_XDR	= 256,565};566 567enum ib_stat_flag {568	IB_STAT_FLAG_OPTIONAL = 1 << 0,569};570 571/**572 * struct rdma_stat_desc573 * @name - The name of the counter574 * @flags - Flags of the counter; For example, IB_STAT_FLAG_OPTIONAL575 * @priv - Driver private information; Core code should not use576 */577struct rdma_stat_desc {578	const char *name;579	unsigned int flags;580	const void *priv;581};582 583/**584 * struct rdma_hw_stats585 * @lock - Mutex to protect parallel write access to lifespan and values586 *    of counters, which are 64bits and not guaranteed to be written587 *    atomicaly on 32bits systems.588 * @timestamp - Used by the core code to track when the last update was589 * @lifespan - Used by the core code to determine how old the counters590 *   should be before being updated again.  Stored in jiffies, defaults591 *   to 10 milliseconds, drivers can override the default be specifying592 *   their own value during their allocation routine.593 * @descs - Array of pointers to static descriptors used for the counters594 *   in directory.595 * @is_disabled - A bitmap to indicate each counter is currently disabled596 *   or not.597 * @num_counters - How many hardware counters there are.  If name is598 *   shorter than this number, a kernel oops will result.  Driver authors599 *   are encouraged to leave BUILD_BUG_ON(ARRAY_SIZE(@name) < num_counters)600 *   in their code to prevent this.601 * @value - Array of u64 counters that are accessed by the sysfs code and602 *   filled in by the drivers get_stats routine603 */604struct rdma_hw_stats {605	struct mutex	lock; /* Protect lifespan and values[] */606	unsigned long	timestamp;607	unsigned long	lifespan;608	const struct rdma_stat_desc *descs;609	unsigned long	*is_disabled;610	int		num_counters;611	u64		value[] __counted_by(num_counters);612};613 614#define RDMA_HW_STATS_DEFAULT_LIFESPAN 10615 616struct rdma_hw_stats *rdma_alloc_hw_stats_struct(617	const struct rdma_stat_desc *descs, int num_counters,618	unsigned long lifespan);619 620void rdma_free_hw_stats_struct(struct rdma_hw_stats *stats);621 622/* Define bits for the various functionality this port needs to be supported by623 * the core.624 */625/* Management                           0x00000FFF */626#define RDMA_CORE_CAP_IB_MAD            0x00000001627#define RDMA_CORE_CAP_IB_SMI            0x00000002628#define RDMA_CORE_CAP_IB_CM             0x00000004629#define RDMA_CORE_CAP_IW_CM             0x00000008630#define RDMA_CORE_CAP_IB_SA             0x00000010631#define RDMA_CORE_CAP_OPA_MAD           0x00000020632 633/* Address format                       0x000FF000 */634#define RDMA_CORE_CAP_AF_IB             0x00001000635#define RDMA_CORE_CAP_ETH_AH            0x00002000636#define RDMA_CORE_CAP_OPA_AH            0x00004000637#define RDMA_CORE_CAP_IB_GRH_REQUIRED   0x00008000638 639/* Protocol                             0xFFF00000 */640#define RDMA_CORE_CAP_PROT_IB           0x00100000641#define RDMA_CORE_CAP_PROT_ROCE         0x00200000642#define RDMA_CORE_CAP_PROT_IWARP        0x00400000643#define RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP 0x00800000644#define RDMA_CORE_CAP_PROT_RAW_PACKET   0x01000000645#define RDMA_CORE_CAP_PROT_USNIC        0x02000000646 647#define RDMA_CORE_PORT_IB_GRH_REQUIRED (RDMA_CORE_CAP_IB_GRH_REQUIRED \648					| RDMA_CORE_CAP_PROT_ROCE     \649					| RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP)650 651#define RDMA_CORE_PORT_IBA_IB          (RDMA_CORE_CAP_PROT_IB  \652					| RDMA_CORE_CAP_IB_MAD \653					| RDMA_CORE_CAP_IB_SMI \654					| RDMA_CORE_CAP_IB_CM  \655					| RDMA_CORE_CAP_IB_SA  \656					| RDMA_CORE_CAP_AF_IB)657#define RDMA_CORE_PORT_IBA_ROCE        (RDMA_CORE_CAP_PROT_ROCE \658					| RDMA_CORE_CAP_IB_MAD  \659					| RDMA_CORE_CAP_IB_CM   \660					| RDMA_CORE_CAP_AF_IB   \661					| RDMA_CORE_CAP_ETH_AH)662#define RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP			\663					(RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP \664					| RDMA_CORE_CAP_IB_MAD  \665					| RDMA_CORE_CAP_IB_CM   \666					| RDMA_CORE_CAP_AF_IB   \667					| RDMA_CORE_CAP_ETH_AH)668#define RDMA_CORE_PORT_IWARP           (RDMA_CORE_CAP_PROT_IWARP \669					| RDMA_CORE_CAP_IW_CM)670#define RDMA_CORE_PORT_INTEL_OPA       (RDMA_CORE_PORT_IBA_IB  \671					| RDMA_CORE_CAP_OPA_MAD)672 673#define RDMA_CORE_PORT_RAW_PACKET	(RDMA_CORE_CAP_PROT_RAW_PACKET)674 675#define RDMA_CORE_PORT_USNIC		(RDMA_CORE_CAP_PROT_USNIC)676 677struct ib_port_attr {678	u64			subnet_prefix;679	enum ib_port_state	state;680	enum ib_mtu		max_mtu;681	enum ib_mtu		active_mtu;682	u32                     phys_mtu;683	int			gid_tbl_len;684	unsigned int		ip_gids:1;685	/* This is the value from PortInfo CapabilityMask, defined by IBA */686	u32			port_cap_flags;687	u32			max_msg_sz;688	u32			bad_pkey_cntr;689	u32			qkey_viol_cntr;690	u16			pkey_tbl_len;691	u32			sm_lid;692	u32			lid;693	u8			lmc;694	u8			max_vl_num;695	u8			sm_sl;696	u8			subnet_timeout;697	u8			init_type_reply;698	u8			active_width;699	u16			active_speed;700	u8                      phys_state;701	u16			port_cap_flags2;702};703 704enum ib_device_modify_flags {705	IB_DEVICE_MODIFY_SYS_IMAGE_GUID	= 1 << 0,706	IB_DEVICE_MODIFY_NODE_DESC	= 1 << 1707};708 709#define IB_DEVICE_NODE_DESC_MAX 64710 711struct ib_device_modify {712	u64	sys_image_guid;713	char	node_desc[IB_DEVICE_NODE_DESC_MAX];714};715 716enum ib_port_modify_flags {717	IB_PORT_SHUTDOWN		= 1,718	IB_PORT_INIT_TYPE		= (1<<2),719	IB_PORT_RESET_QKEY_CNTR		= (1<<3),720	IB_PORT_OPA_MASK_CHG		= (1<<4)721};722 723struct ib_port_modify {724	u32	set_port_cap_mask;725	u32	clr_port_cap_mask;726	u8	init_type;727};728 729enum ib_event_type {730	IB_EVENT_CQ_ERR,731	IB_EVENT_QP_FATAL,732	IB_EVENT_QP_REQ_ERR,733	IB_EVENT_QP_ACCESS_ERR,734	IB_EVENT_COMM_EST,735	IB_EVENT_SQ_DRAINED,736	IB_EVENT_PATH_MIG,737	IB_EVENT_PATH_MIG_ERR,738	IB_EVENT_DEVICE_FATAL,739	IB_EVENT_PORT_ACTIVE,740	IB_EVENT_PORT_ERR,741	IB_EVENT_LID_CHANGE,742	IB_EVENT_PKEY_CHANGE,743	IB_EVENT_SM_CHANGE,744	IB_EVENT_SRQ_ERR,745	IB_EVENT_SRQ_LIMIT_REACHED,746	IB_EVENT_QP_LAST_WQE_REACHED,747	IB_EVENT_CLIENT_REREGISTER,748	IB_EVENT_GID_CHANGE,749	IB_EVENT_WQ_FATAL,750};751 752const char *__attribute_const__ ib_event_msg(enum ib_event_type event);753 754struct ib_event {755	struct ib_device	*device;756	union {757		struct ib_cq	*cq;758		struct ib_qp	*qp;759		struct ib_srq	*srq;760		struct ib_wq	*wq;761		u32		port_num;762	} element;763	enum ib_event_type	event;764};765 766struct ib_event_handler {767	struct ib_device *device;768	void            (*handler)(struct ib_event_handler *, struct ib_event *);769	struct list_head  list;770};771 772#define INIT_IB_EVENT_HANDLER(_ptr, _device, _handler)		\773	do {							\774		(_ptr)->device  = _device;			\775		(_ptr)->handler = _handler;			\776		INIT_LIST_HEAD(&(_ptr)->list);			\777	} while (0)778 779struct ib_global_route {780	const struct ib_gid_attr *sgid_attr;781	union ib_gid	dgid;782	u32		flow_label;783	u8		sgid_index;784	u8		hop_limit;785	u8		traffic_class;786};787 788struct ib_grh {789	__be32		version_tclass_flow;790	__be16		paylen;791	u8		next_hdr;792	u8		hop_limit;793	union ib_gid	sgid;794	union ib_gid	dgid;795};796 797union rdma_network_hdr {798	struct ib_grh ibgrh;799	struct {800		/* The IB spec states that if it's IPv4, the header801		 * is located in the last 20 bytes of the header.802		 */803		u8		reserved[20];804		struct iphdr	roce4grh;805	};806};807 808#define IB_QPN_MASK		0xFFFFFF809 810enum {811	IB_MULTICAST_QPN = 0xffffff812};813 814#define IB_LID_PERMISSIVE	cpu_to_be16(0xFFFF)815#define IB_MULTICAST_LID_BASE	cpu_to_be16(0xC000)816 817enum ib_ah_flags {818	IB_AH_GRH	= 1819};820 821enum ib_rate {822	IB_RATE_PORT_CURRENT = 0,823	IB_RATE_2_5_GBPS = 2,824	IB_RATE_5_GBPS   = 5,825	IB_RATE_10_GBPS  = 3,826	IB_RATE_20_GBPS  = 6,827	IB_RATE_30_GBPS  = 4,828	IB_RATE_40_GBPS  = 7,829	IB_RATE_60_GBPS  = 8,830	IB_RATE_80_GBPS  = 9,831	IB_RATE_120_GBPS = 10,832	IB_RATE_14_GBPS  = 11,833	IB_RATE_56_GBPS  = 12,834	IB_RATE_112_GBPS = 13,835	IB_RATE_168_GBPS = 14,836	IB_RATE_25_GBPS  = 15,837	IB_RATE_100_GBPS = 16,838	IB_RATE_200_GBPS = 17,839	IB_RATE_300_GBPS = 18,840	IB_RATE_28_GBPS  = 19,841	IB_RATE_50_GBPS  = 20,842	IB_RATE_400_GBPS = 21,843	IB_RATE_600_GBPS = 22,844	IB_RATE_800_GBPS = 23,845};846 847/**848 * ib_rate_to_mult - Convert the IB rate enum to a multiple of the849 * base rate of 2.5 Gbit/sec.  For example, IB_RATE_5_GBPS will be850 * converted to 2, since 5 Gbit/sec is 2 * 2.5 Gbit/sec.851 * @rate: rate to convert.852 */853__attribute_const__ int ib_rate_to_mult(enum ib_rate rate);854 855/**856 * ib_rate_to_mbps - Convert the IB rate enum to Mbps.857 * For example, IB_RATE_2_5_GBPS will be converted to 2500.858 * @rate: rate to convert.859 */860__attribute_const__ int ib_rate_to_mbps(enum ib_rate rate);861 862 863/**864 * enum ib_mr_type - memory region type865 * @IB_MR_TYPE_MEM_REG:       memory region that is used for866 *                            normal registration867 * @IB_MR_TYPE_SG_GAPS:       memory region that is capable to868 *                            register any arbitrary sg lists (without869 *                            the normal mr constraints - see870 *                            ib_map_mr_sg)871 * @IB_MR_TYPE_DM:            memory region that is used for device872 *                            memory registration873 * @IB_MR_TYPE_USER:          memory region that is used for the user-space874 *                            application875 * @IB_MR_TYPE_DMA:           memory region that is used for DMA operations876 *                            without address translations (VA=PA)877 * @IB_MR_TYPE_INTEGRITY:     memory region that is used for878 *                            data integrity operations879 */880enum ib_mr_type {881	IB_MR_TYPE_MEM_REG,882	IB_MR_TYPE_SG_GAPS,883	IB_MR_TYPE_DM,884	IB_MR_TYPE_USER,885	IB_MR_TYPE_DMA,886	IB_MR_TYPE_INTEGRITY,887};888 889enum ib_mr_status_check {890	IB_MR_CHECK_SIG_STATUS = 1,891};892 893/**894 * struct ib_mr_status - Memory region status container895 *896 * @fail_status: Bitmask of MR checks status. For each897 *     failed check a corresponding status bit is set.898 * @sig_err: Additional info for IB_MR_CEHCK_SIG_STATUS899 *     failure.900 */901struct ib_mr_status {902	u32		    fail_status;903	struct ib_sig_err   sig_err;904};905 906/**907 * mult_to_ib_rate - Convert a multiple of 2.5 Gbit/sec to an IB rate908 * enum.909 * @mult: multiple to convert.910 */911__attribute_const__ enum ib_rate mult_to_ib_rate(int mult);912 913struct rdma_ah_init_attr {914	struct rdma_ah_attr *ah_attr;915	u32 flags;916	struct net_device *xmit_slave;917};918 919enum rdma_ah_attr_type {920	RDMA_AH_ATTR_TYPE_UNDEFINED,921	RDMA_AH_ATTR_TYPE_IB,922	RDMA_AH_ATTR_TYPE_ROCE,923	RDMA_AH_ATTR_TYPE_OPA,924};925 926struct ib_ah_attr {927	u16			dlid;928	u8			src_path_bits;929};930 931struct roce_ah_attr {932	u8			dmac[ETH_ALEN];933};934 935struct opa_ah_attr {936	u32			dlid;937	u8			src_path_bits;938	bool			make_grd;939};940 941struct rdma_ah_attr {942	struct ib_global_route	grh;943	u8			sl;944	u8			static_rate;945	u32			port_num;946	u8			ah_flags;947	enum rdma_ah_attr_type type;948	union {949		struct ib_ah_attr ib;950		struct roce_ah_attr roce;951		struct opa_ah_attr opa;952	};953};954 955enum ib_wc_status {956	IB_WC_SUCCESS,957	IB_WC_LOC_LEN_ERR,958	IB_WC_LOC_QP_OP_ERR,959	IB_WC_LOC_EEC_OP_ERR,960	IB_WC_LOC_PROT_ERR,961	IB_WC_WR_FLUSH_ERR,962	IB_WC_MW_BIND_ERR,963	IB_WC_BAD_RESP_ERR,964	IB_WC_LOC_ACCESS_ERR,965	IB_WC_REM_INV_REQ_ERR,966	IB_WC_REM_ACCESS_ERR,967	IB_WC_REM_OP_ERR,968	IB_WC_RETRY_EXC_ERR,969	IB_WC_RNR_RETRY_EXC_ERR,970	IB_WC_LOC_RDD_VIOL_ERR,971	IB_WC_REM_INV_RD_REQ_ERR,972	IB_WC_REM_ABORT_ERR,973	IB_WC_INV_EECN_ERR,974	IB_WC_INV_EEC_STATE_ERR,975	IB_WC_FATAL_ERR,976	IB_WC_RESP_TIMEOUT_ERR,977	IB_WC_GENERAL_ERR978};979 980const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status);981 982enum ib_wc_opcode {983	IB_WC_SEND = IB_UVERBS_WC_SEND,984	IB_WC_RDMA_WRITE = IB_UVERBS_WC_RDMA_WRITE,985	IB_WC_RDMA_READ = IB_UVERBS_WC_RDMA_READ,986	IB_WC_COMP_SWAP = IB_UVERBS_WC_COMP_SWAP,987	IB_WC_FETCH_ADD = IB_UVERBS_WC_FETCH_ADD,988	IB_WC_BIND_MW = IB_UVERBS_WC_BIND_MW,989	IB_WC_LOCAL_INV = IB_UVERBS_WC_LOCAL_INV,990	IB_WC_LSO = IB_UVERBS_WC_TSO,991	IB_WC_ATOMIC_WRITE = IB_UVERBS_WC_ATOMIC_WRITE,992	IB_WC_REG_MR,993	IB_WC_MASKED_COMP_SWAP,994	IB_WC_MASKED_FETCH_ADD,995	IB_WC_FLUSH = IB_UVERBS_WC_FLUSH,996/*997 * Set value of IB_WC_RECV so consumers can test if a completion is a998 * receive by testing (opcode & IB_WC_RECV).999 */1000	IB_WC_RECV			= 1 << 7,1001	IB_WC_RECV_RDMA_WITH_IMM1002};1003 1004enum ib_wc_flags {1005	IB_WC_GRH		= 1,1006	IB_WC_WITH_IMM		= (1<<1),1007	IB_WC_WITH_INVALIDATE	= (1<<2),1008	IB_WC_IP_CSUM_OK	= (1<<3),1009	IB_WC_WITH_SMAC		= (1<<4),1010	IB_WC_WITH_VLAN		= (1<<5),1011	IB_WC_WITH_NETWORK_HDR_TYPE	= (1<<6),1012};1013 1014struct ib_wc {1015	union {1016		u64		wr_id;1017		struct ib_cqe	*wr_cqe;1018	};1019	enum ib_wc_status	status;1020	enum ib_wc_opcode	opcode;1021	u32			vendor_err;1022	u32			byte_len;1023	struct ib_qp	       *qp;1024	union {1025		__be32		imm_data;1026		u32		invalidate_rkey;1027	} ex;1028	u32			src_qp;1029	u32			slid;1030	int			wc_flags;1031	u16			pkey_index;1032	u8			sl;1033	u8			dlid_path_bits;1034	u32 port_num; /* valid only for DR SMPs on switches */1035	u8			smac[ETH_ALEN];1036	u16			vlan_id;1037	u8			network_hdr_type;1038};1039 1040enum ib_cq_notify_flags {1041	IB_CQ_SOLICITED			= 1 << 0,1042	IB_CQ_NEXT_COMP			= 1 << 1,1043	IB_CQ_SOLICITED_MASK		= IB_CQ_SOLICITED | IB_CQ_NEXT_COMP,1044	IB_CQ_REPORT_MISSED_EVENTS	= 1 << 2,1045};1046 1047enum ib_srq_type {1048	IB_SRQT_BASIC = IB_UVERBS_SRQT_BASIC,1049	IB_SRQT_XRC = IB_UVERBS_SRQT_XRC,1050	IB_SRQT_TM = IB_UVERBS_SRQT_TM,1051};1052 1053static inline bool ib_srq_has_cq(enum ib_srq_type srq_type)1054{1055	return srq_type == IB_SRQT_XRC ||1056	       srq_type == IB_SRQT_TM;1057}1058 1059enum ib_srq_attr_mask {1060	IB_SRQ_MAX_WR	= 1 << 0,1061	IB_SRQ_LIMIT	= 1 << 1,1062};1063 1064struct ib_srq_attr {1065	u32	max_wr;1066	u32	max_sge;1067	u32	srq_limit;1068};1069 1070struct ib_srq_init_attr {1071	void		      (*event_handler)(struct ib_event *, void *);1072	void		       *srq_context;1073	struct ib_srq_attr	attr;1074	enum ib_srq_type	srq_type;1075 1076	struct {1077		struct ib_cq   *cq;1078		union {1079			struct {1080				struct ib_xrcd *xrcd;1081			} xrc;1082 1083			struct {1084				u32		max_num_tags;1085			} tag_matching;1086		};1087	} ext;1088};1089 1090struct ib_qp_cap {1091	u32	max_send_wr;1092	u32	max_recv_wr;1093	u32	max_send_sge;1094	u32	max_recv_sge;1095	u32	max_inline_data;1096 1097	/*1098	 * Maximum number of rdma_rw_ctx structures in flight at a time.1099	 * ib_create_qp() will calculate the right amount of needed WRs1100	 * and MRs based on this.1101	 */1102	u32	max_rdma_ctxs;1103};1104 1105enum ib_sig_type {1106	IB_SIGNAL_ALL_WR,1107	IB_SIGNAL_REQ_WR1108};1109 1110enum ib_qp_type {1111	/*1112	 * IB_QPT_SMI and IB_QPT_GSI have to be the first two entries1113	 * here (and in that order) since the MAD layer uses them as1114	 * indices into a 2-entry table.1115	 */1116	IB_QPT_SMI,1117	IB_QPT_GSI,1118 1119	IB_QPT_RC = IB_UVERBS_QPT_RC,1120	IB_QPT_UC = IB_UVERBS_QPT_UC,1121	IB_QPT_UD = IB_UVERBS_QPT_UD,1122	IB_QPT_RAW_IPV6,1123	IB_QPT_RAW_ETHERTYPE,1124	IB_QPT_RAW_PACKET = IB_UVERBS_QPT_RAW_PACKET,1125	IB_QPT_XRC_INI = IB_UVERBS_QPT_XRC_INI,1126	IB_QPT_XRC_TGT = IB_UVERBS_QPT_XRC_TGT,1127	IB_QPT_MAX,1128	IB_QPT_DRIVER = IB_UVERBS_QPT_DRIVER,1129	/* Reserve a range for qp types internal to the low level driver.1130	 * These qp types will not be visible at the IB core layer, so the1131	 * IB_QPT_MAX usages should not be affected in the core layer1132	 */1133	IB_QPT_RESERVED1 = 0x1000,1134	IB_QPT_RESERVED2,1135	IB_QPT_RESERVED3,1136	IB_QPT_RESERVED4,1137	IB_QPT_RESERVED5,1138	IB_QPT_RESERVED6,1139	IB_QPT_RESERVED7,1140	IB_QPT_RESERVED8,1141	IB_QPT_RESERVED9,1142	IB_QPT_RESERVED10,1143};1144 1145enum ib_qp_create_flags {1146	IB_QP_CREATE_IPOIB_UD_LSO		= 1 << 0,1147	IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK	=1148		IB_UVERBS_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,1149	IB_QP_CREATE_CROSS_CHANNEL              = 1 << 2,1150	IB_QP_CREATE_MANAGED_SEND               = 1 << 3,1151	IB_QP_CREATE_MANAGED_RECV               = 1 << 4,1152	IB_QP_CREATE_NETIF_QP			= 1 << 5,1153	IB_QP_CREATE_INTEGRITY_EN		= 1 << 6,1154	IB_QP_CREATE_NETDEV_USE			= 1 << 7,1155	IB_QP_CREATE_SCATTER_FCS		=1156		IB_UVERBS_QP_CREATE_SCATTER_FCS,1157	IB_QP_CREATE_CVLAN_STRIPPING		=1158		IB_UVERBS_QP_CREATE_CVLAN_STRIPPING,1159	IB_QP_CREATE_SOURCE_QPN			= 1 << 10,1160	IB_QP_CREATE_PCI_WRITE_END_PADDING	=1161		IB_UVERBS_QP_CREATE_PCI_WRITE_END_PADDING,1162	/* reserve bits 26-31 for low level drivers' internal use */1163	IB_QP_CREATE_RESERVED_START		= 1 << 26,1164	IB_QP_CREATE_RESERVED_END		= 1 << 31,1165};1166 1167/*1168 * Note: users may not call ib_close_qp or ib_destroy_qp from the event_handler1169 * callback to destroy the passed in QP.1170 */1171 1172struct ib_qp_init_attr {1173	/* This callback occurs in workqueue context */1174	void                  (*event_handler)(struct ib_event *, void *);1175 1176	void		       *qp_context;1177	struct ib_cq	       *send_cq;1178	struct ib_cq	       *recv_cq;1179	struct ib_srq	       *srq;1180	struct ib_xrcd	       *xrcd;     /* XRC TGT QPs only */1181	struct ib_qp_cap	cap;1182	enum ib_sig_type	sq_sig_type;1183	enum ib_qp_type		qp_type;1184	u32			create_flags;1185 1186	/*1187	 * Only needed for special QP types, or when using the RW API.1188	 */1189	u32			port_num;1190	struct ib_rwq_ind_table *rwq_ind_tbl;1191	u32			source_qpn;1192};1193 1194struct ib_qp_open_attr {1195	void                  (*event_handler)(struct ib_event *, void *);1196	void		       *qp_context;1197	u32			qp_num;1198	enum ib_qp_type		qp_type;1199};1200 1201enum ib_rnr_timeout {1202	IB_RNR_TIMER_655_36 =  0,1203	IB_RNR_TIMER_000_01 =  1,1204	IB_RNR_TIMER_000_02 =  2,1205	IB_RNR_TIMER_000_03 =  3,1206	IB_RNR_TIMER_000_04 =  4,1207	IB_RNR_TIMER_000_06 =  5,1208	IB_RNR_TIMER_000_08 =  6,1209	IB_RNR_TIMER_000_12 =  7,1210	IB_RNR_TIMER_000_16 =  8,1211	IB_RNR_TIMER_000_24 =  9,1212	IB_RNR_TIMER_000_32 = 10,1213	IB_RNR_TIMER_000_48 = 11,1214	IB_RNR_TIMER_000_64 = 12,1215	IB_RNR_TIMER_000_96 = 13,1216	IB_RNR_TIMER_001_28 = 14,1217	IB_RNR_TIMER_001_92 = 15,1218	IB_RNR_TIMER_002_56 = 16,1219	IB_RNR_TIMER_003_84 = 17,1220	IB_RNR_TIMER_005_12 = 18,1221	IB_RNR_TIMER_007_68 = 19,1222	IB_RNR_TIMER_010_24 = 20,1223	IB_RNR_TIMER_015_36 = 21,1224	IB_RNR_TIMER_020_48 = 22,1225	IB_RNR_TIMER_030_72 = 23,1226	IB_RNR_TIMER_040_96 = 24,1227	IB_RNR_TIMER_061_44 = 25,1228	IB_RNR_TIMER_081_92 = 26,1229	IB_RNR_TIMER_122_88 = 27,1230	IB_RNR_TIMER_163_84 = 28,1231	IB_RNR_TIMER_245_76 = 29,1232	IB_RNR_TIMER_327_68 = 30,1233	IB_RNR_TIMER_491_52 = 311234};1235 1236enum ib_qp_attr_mask {1237	IB_QP_STATE			= 1,1238	IB_QP_CUR_STATE			= (1<<1),1239	IB_QP_EN_SQD_ASYNC_NOTIFY	= (1<<2),1240	IB_QP_ACCESS_FLAGS		= (1<<3),1241	IB_QP_PKEY_INDEX		= (1<<4),1242	IB_QP_PORT			= (1<<5),1243	IB_QP_QKEY			= (1<<6),1244	IB_QP_AV			= (1<<7),1245	IB_QP_PATH_MTU			= (1<<8),1246	IB_QP_TIMEOUT			= (1<<9),1247	IB_QP_RETRY_CNT			= (1<<10),1248	IB_QP_RNR_RETRY			= (1<<11),1249	IB_QP_RQ_PSN			= (1<<12),1250	IB_QP_MAX_QP_RD_ATOMIC		= (1<<13),1251	IB_QP_ALT_PATH			= (1<<14),1252	IB_QP_MIN_RNR_TIMER		= (1<<15),1253	IB_QP_SQ_PSN			= (1<<16),1254	IB_QP_MAX_DEST_RD_ATOMIC	= (1<<17),1255	IB_QP_PATH_MIG_STATE		= (1<<18),1256	IB_QP_CAP			= (1<<19),1257	IB_QP_DEST_QPN			= (1<<20),1258	IB_QP_RESERVED1			= (1<<21),1259	IB_QP_RESERVED2			= (1<<22),1260	IB_QP_RESERVED3			= (1<<23),1261	IB_QP_RESERVED4			= (1<<24),1262	IB_QP_RATE_LIMIT		= (1<<25),1263 1264	IB_QP_ATTR_STANDARD_BITS = GENMASK(20, 0),1265};1266 1267enum ib_qp_state {1268	IB_QPS_RESET,1269	IB_QPS_INIT,1270	IB_QPS_RTR,1271	IB_QPS_RTS,1272	IB_QPS_SQD,1273	IB_QPS_SQE,1274	IB_QPS_ERR1275};1276 1277enum ib_mig_state {1278	IB_MIG_MIGRATED,1279	IB_MIG_REARM,1280	IB_MIG_ARMED1281};1282 1283enum ib_mw_type {1284	IB_MW_TYPE_1 = 1,1285	IB_MW_TYPE_2 = 21286};1287 1288struct ib_qp_attr {1289	enum ib_qp_state	qp_state;1290	enum ib_qp_state	cur_qp_state;1291	enum ib_mtu		path_mtu;1292	enum ib_mig_state	path_mig_state;1293	u32			qkey;1294	u32			rq_psn;1295	u32			sq_psn;1296	u32			dest_qp_num;1297	int			qp_access_flags;1298	struct ib_qp_cap	cap;1299	struct rdma_ah_attr	ah_attr;1300	struct rdma_ah_attr	alt_ah_attr;1301	u16			pkey_index;1302	u16			alt_pkey_index;1303	u8			en_sqd_async_notify;1304	u8			sq_draining;1305	u8			max_rd_atomic;1306	u8			max_dest_rd_atomic;1307	u8			min_rnr_timer;1308	u32			port_num;1309	u8			timeout;1310	u8			retry_cnt;1311	u8			rnr_retry;1312	u32			alt_port_num;1313	u8			alt_timeout;1314	u32			rate_limit;1315	struct net_device	*xmit_slave;1316};1317 1318enum ib_wr_opcode {1319	/* These are shared with userspace */1320	IB_WR_RDMA_WRITE = IB_UVERBS_WR_RDMA_WRITE,1321	IB_WR_RDMA_WRITE_WITH_IMM = IB_UVERBS_WR_RDMA_WRITE_WITH_IMM,1322	IB_WR_SEND = IB_UVERBS_WR_SEND,1323	IB_WR_SEND_WITH_IMM = IB_UVERBS_WR_SEND_WITH_IMM,1324	IB_WR_RDMA_READ = IB_UVERBS_WR_RDMA_READ,1325	IB_WR_ATOMIC_CMP_AND_SWP = IB_UVERBS_WR_ATOMIC_CMP_AND_SWP,1326	IB_WR_ATOMIC_FETCH_AND_ADD = IB_UVERBS_WR_ATOMIC_FETCH_AND_ADD,1327	IB_WR_BIND_MW = IB_UVERBS_WR_BIND_MW,1328	IB_WR_LSO = IB_UVERBS_WR_TSO,1329	IB_WR_SEND_WITH_INV = IB_UVERBS_WR_SEND_WITH_INV,1330	IB_WR_RDMA_READ_WITH_INV = IB_UVERBS_WR_RDMA_READ_WITH_INV,1331	IB_WR_LOCAL_INV = IB_UVERBS_WR_LOCAL_INV,1332	IB_WR_MASKED_ATOMIC_CMP_AND_SWP =1333		IB_UVERBS_WR_MASKED_ATOMIC_CMP_AND_SWP,1334	IB_WR_MASKED_ATOMIC_FETCH_AND_ADD =1335		IB_UVERBS_WR_MASKED_ATOMIC_FETCH_AND_ADD,1336	IB_WR_FLUSH = IB_UVERBS_WR_FLUSH,1337	IB_WR_ATOMIC_WRITE = IB_UVERBS_WR_ATOMIC_WRITE,1338 1339	/* These are kernel only and can not be issued by userspace */1340	IB_WR_REG_MR = 0x20,1341	IB_WR_REG_MR_INTEGRITY,1342 1343	/* reserve values for low level drivers' internal use.1344	 * These values will not be used at all in the ib core layer.1345	 */1346	IB_WR_RESERVED1 = 0xf0,1347	IB_WR_RESERVED2,1348	IB_WR_RESERVED3,1349	IB_WR_RESERVED4,1350	IB_WR_RESERVED5,1351	IB_WR_RESERVED6,1352	IB_WR_RESERVED7,1353	IB_WR_RESERVED8,1354	IB_WR_RESERVED9,1355	IB_WR_RESERVED10,1356};1357 1358enum ib_send_flags {1359	IB_SEND_FENCE		= 1,1360	IB_SEND_SIGNALED	= (1<<1),1361	IB_SEND_SOLICITED	= (1<<2),1362	IB_SEND_INLINE		= (1<<3),1363	IB_SEND_IP_CSUM		= (1<<4),1364 1365	/* reserve bits 26-31 for low level drivers' internal use */1366	IB_SEND_RESERVED_START	= (1 << 26),1367	IB_SEND_RESERVED_END	= (1 << 31),1368};1369 1370struct ib_sge {1371	u64	addr;1372	u32	length;1373	u32	lkey;1374};1375 1376struct ib_cqe {1377	void (*done)(struct ib_cq *cq, struct ib_wc *wc);1378};1379 1380struct ib_send_wr {1381	struct ib_send_wr      *next;1382	union {1383		u64		wr_id;1384		struct ib_cqe	*wr_cqe;1385	};1386	struct ib_sge	       *sg_list;1387	int			num_sge;1388	enum ib_wr_opcode	opcode;1389	int			send_flags;1390	union {1391		__be32		imm_data;1392		u32		invalidate_rkey;1393	} ex;1394};1395 1396struct ib_rdma_wr {1397	struct ib_send_wr	wr;1398	u64			remote_addr;1399	u32			rkey;1400};1401 1402static inline const struct ib_rdma_wr *rdma_wr(const struct ib_send_wr *wr)1403{1404	return container_of(wr, struct ib_rdma_wr, wr);1405}1406 1407struct ib_atomic_wr {1408	struct ib_send_wr	wr;1409	u64			remote_addr;1410	u64			compare_add;1411	u64			swap;1412	u64			compare_add_mask;1413	u64			swap_mask;1414	u32			rkey;1415};1416 1417static inline const struct ib_atomic_wr *atomic_wr(const struct ib_send_wr *wr)1418{1419	return container_of(wr, struct ib_atomic_wr, wr);1420}1421 1422struct ib_ud_wr {1423	struct ib_send_wr	wr;1424	struct ib_ah		*ah;1425	void			*header;1426	int			hlen;1427	int			mss;1428	u32			remote_qpn;1429	u32			remote_qkey;1430	u16			pkey_index; /* valid for GSI only */1431	u32			port_num; /* valid for DR SMPs on switch only */1432};1433 1434static inline const struct ib_ud_wr *ud_wr(const struct ib_send_wr *wr)1435{1436	return container_of(wr, struct ib_ud_wr, wr);1437}1438 1439struct ib_reg_wr {1440	struct ib_send_wr	wr;1441	struct ib_mr		*mr;1442	u32			key;1443	int			access;1444};1445 1446static inline const struct ib_reg_wr *reg_wr(const struct ib_send_wr *wr)1447{1448	return container_of(wr, struct ib_reg_wr, wr);1449}1450 1451struct ib_recv_wr {1452	struct ib_recv_wr      *next;1453	union {1454		u64		wr_id;1455		struct ib_cqe	*wr_cqe;1456	};1457	struct ib_sge	       *sg_list;1458	int			num_sge;1459};1460 1461enum ib_access_flags {1462	IB_ACCESS_LOCAL_WRITE = IB_UVERBS_ACCESS_LOCAL_WRITE,1463	IB_ACCESS_REMOTE_WRITE = IB_UVERBS_ACCESS_REMOTE_WRITE,1464	IB_ACCESS_REMOTE_READ = IB_UVERBS_ACCESS_REMOTE_READ,1465	IB_ACCESS_REMOTE_ATOMIC = IB_UVERBS_ACCESS_REMOTE_ATOMIC,1466	IB_ACCESS_MW_BIND = IB_UVERBS_ACCESS_MW_BIND,1467	IB_ZERO_BASED = IB_UVERBS_ACCESS_ZERO_BASED,1468	IB_ACCESS_ON_DEMAND = IB_UVERBS_ACCESS_ON_DEMAND,1469	IB_ACCESS_HUGETLB = IB_UVERBS_ACCESS_HUGETLB,1470	IB_ACCESS_RELAXED_ORDERING = IB_UVERBS_ACCESS_RELAXED_ORDERING,1471	IB_ACCESS_FLUSH_GLOBAL = IB_UVERBS_ACCESS_FLUSH_GLOBAL,1472	IB_ACCESS_FLUSH_PERSISTENT = IB_UVERBS_ACCESS_FLUSH_PERSISTENT,1473 1474	IB_ACCESS_OPTIONAL = IB_UVERBS_ACCESS_OPTIONAL_RANGE,1475	IB_ACCESS_SUPPORTED =1476		((IB_ACCESS_FLUSH_PERSISTENT << 1) - 1) | IB_ACCESS_OPTIONAL,1477};1478 1479/*1480 * XXX: these are apparently used for ->rereg_user_mr, no idea why they1481 * are hidden here instead of a uapi header!1482 */1483enum ib_mr_rereg_flags {1484	IB_MR_REREG_TRANS	= 1,1485	IB_MR_REREG_PD		= (1<<1),1486	IB_MR_REREG_ACCESS	= (1<<2),1487	IB_MR_REREG_SUPPORTED	= ((IB_MR_REREG_ACCESS << 1) - 1)1488};1489 1490struct ib_umem;1491 1492enum rdma_remove_reason {1493	/*1494	 * Userspace requested uobject deletion or initial try1495	 * to remove uobject via cleanup. Call could fail1496	 */1497	RDMA_REMOVE_DESTROY,1498	/* Context deletion. This call should delete the actual object itself */1499	RDMA_REMOVE_CLOSE,1500	/* Driver is being hot-unplugged. This call should delete the actual object itself */1501	RDMA_REMOVE_DRIVER_REMOVE,1502	/* uobj is being cleaned-up before being committed */1503	RDMA_REMOVE_ABORT,1504	/* The driver failed to destroy the uobject and is being disconnected */1505	RDMA_REMOVE_DRIVER_FAILURE,1506};1507 1508struct ib_rdmacg_object {1509#ifdef CONFIG_CGROUP_RDMA1510	struct rdma_cgroup	*cg;		/* owner rdma cgroup */1511#endif1512};1513 1514struct ib_ucontext {1515	struct ib_device       *device;1516	struct ib_uverbs_file  *ufile;1517 1518	struct ib_rdmacg_object	cg_obj;1519	/*1520	 * Implementation details of the RDMA core, don't use in drivers:1521	 */1522	struct rdma_restrack_entry res;1523	struct xarray mmap_xa;1524};1525 1526struct ib_uobject {1527	u64			user_handle;	/* handle given to us by userspace */1528	/* ufile & ucontext owning this object */1529	struct ib_uverbs_file  *ufile;1530	/* FIXME, save memory: ufile->context == context */1531	struct ib_ucontext     *context;	/* associated user context */1532	void		       *object;		/* containing object */1533	struct list_head	list;		/* link to context's list */1534	struct ib_rdmacg_object	cg_obj;		/* rdmacg object */1535	int			id;		/* index into kernel idr */1536	struct kref		ref;1537	atomic_t		usecnt;		/* protects exclusive access */1538	struct rcu_head		rcu;		/* kfree_rcu() overhead */1539 1540	const struct uverbs_api_object *uapi_object;1541};1542 1543struct ib_udata {1544	const void __user *inbuf;1545	void __user *outbuf;1546	size_t       inlen;1547	size_t       outlen;1548};1549 1550struct ib_pd {1551	u32			local_dma_lkey;1552	u32			flags;1553	struct ib_device       *device;1554	struct ib_uobject      *uobject;1555	atomic_t          	usecnt; /* count all resources */1556 1557	u32			unsafe_global_rkey;1558 1559	/*1560	 * Implementation details of the RDMA core, don't use in drivers:1561	 */1562	struct ib_mr	       *__internal_mr;1563	struct rdma_restrack_entry res;1564};1565 1566struct ib_xrcd {1567	struct ib_device       *device;1568	atomic_t		usecnt; /* count all exposed resources */1569	struct inode	       *inode;1570	struct rw_semaphore	tgt_qps_rwsem;1571	struct xarray		tgt_qps;1572};1573 1574struct ib_ah {1575	struct ib_device	*device;1576	struct ib_pd		*pd;1577	struct ib_uobject	*uobject;1578	const struct ib_gid_attr *sgid_attr;1579	enum rdma_ah_attr_type	type;1580};1581 1582typedef void (*ib_comp_handler)(struct ib_cq *cq, void *cq_context);1583 1584enum ib_poll_context {1585	IB_POLL_SOFTIRQ,	   /* poll from softirq context */1586	IB_POLL_WORKQUEUE,	   /* poll from workqueue */1587	IB_POLL_UNBOUND_WORKQUEUE, /* poll from unbound workqueue */1588	IB_POLL_LAST_POOL_TYPE = IB_POLL_UNBOUND_WORKQUEUE,1589 1590	IB_POLL_DIRECT,		   /* caller context, no hw completions */1591};1592 1593struct ib_cq {1594	struct ib_device       *device;1595	struct ib_ucq_object   *uobject;1596	ib_comp_handler   	comp_handler;1597	void                  (*event_handler)(struct ib_event *, void *);1598	void                   *cq_context;1599	int               	cqe;1600	unsigned int		cqe_used;1601	atomic_t          	usecnt; /* count number of work queues */1602	enum ib_poll_context	poll_ctx;1603	struct ib_wc		*wc;1604	struct list_head        pool_entry;1605	union {1606		struct irq_poll		iop;1607		struct work_struct	work;1608	};1609	struct workqueue_struct *comp_wq;1610	struct dim *dim;1611 1612	/* updated only by trace points */1613	ktime_t timestamp;1614	u8 interrupt:1;1615	u8 shared:1;1616	unsigned int comp_vector;1617 1618	/*1619	 * Implementation details of the RDMA core, don't use in drivers:1620	 */1621	struct rdma_restrack_entry res;1622};1623 1624struct ib_srq {1625	struct ib_device       *device;1626	struct ib_pd	       *pd;1627	struct ib_usrq_object  *uobject;1628	void		      (*event_handler)(struct ib_event *, void *);1629	void		       *srq_context;1630	enum ib_srq_type	srq_type;1631	atomic_t		usecnt;1632 1633	struct {1634		struct ib_cq   *cq;1635		union {1636			struct {1637				struct ib_xrcd *xrcd;1638				u32		srq_num;1639			} xrc;1640		};1641	} ext;1642 1643	/*1644	 * Implementation details of the RDMA core, don't use in drivers:1645	 */1646	struct rdma_restrack_entry res;1647};1648 1649enum ib_raw_packet_caps {1650	/*1651	 * Strip cvlan from incoming packet and report it in the matching work1652	 * completion is supported.1653	 */1654	IB_RAW_PACKET_CAP_CVLAN_STRIPPING =1655		IB_UVERBS_RAW_PACKET_CAP_CVLAN_STRIPPING,1656	/*1657	 * Scatter FCS field of an incoming packet to host memory is supported.1658	 */1659	IB_RAW_PACKET_CAP_SCATTER_FCS = IB_UVERBS_RAW_PACKET_CAP_SCATTER_FCS,1660	/* Checksum offloads are supported (for both send and receive). */1661	IB_RAW_PACKET_CAP_IP_CSUM = IB_UVERBS_RAW_PACKET_CAP_IP_CSUM,1662	/*1663	 * When a packet is received for an RQ with no receive WQEs, the1664	 * packet processing is delayed.1665	 */1666	IB_RAW_PACKET_CAP_DELAY_DROP = IB_UVERBS_RAW_PACKET_CAP_DELAY_DROP,1667};1668 1669enum ib_wq_type {1670	IB_WQT_RQ = IB_UVERBS_WQT_RQ,1671};1672 1673enum ib_wq_state {1674	IB_WQS_RESET,1675	IB_WQS_RDY,1676	IB_WQS_ERR1677};1678 1679struct ib_wq {1680	struct ib_device       *device;1681	struct ib_uwq_object   *uobject;1682	void		    *wq_context;1683	void		    (*event_handler)(struct ib_event *, void *);1684	struct ib_pd	       *pd;1685	struct ib_cq	       *cq;1686	u32		wq_num;1687	enum ib_wq_state       state;1688	enum ib_wq_type	wq_type;1689	atomic_t		usecnt;1690};1691 1692enum ib_wq_flags {1693	IB_WQ_FLAGS_CVLAN_STRIPPING	= IB_UVERBS_WQ_FLAGS_CVLAN_STRIPPING,1694	IB_WQ_FLAGS_SCATTER_FCS		= IB_UVERBS_WQ_FLAGS_SCATTER_FCS,1695	IB_WQ_FLAGS_DELAY_DROP		= IB_UVERBS_WQ_FLAGS_DELAY_DROP,1696	IB_WQ_FLAGS_PCI_WRITE_END_PADDING =1697				IB_UVERBS_WQ_FLAGS_PCI_WRITE_END_PADDING,1698};1699 1700struct ib_wq_init_attr {1701	void		       *wq_context;1702	enum ib_wq_type	wq_type;1703	u32		max_wr;1704	u32		max_sge;1705	struct	ib_cq	       *cq;1706	void		    (*event_handler)(struct ib_event *, void *);1707	u32		create_flags; /* Use enum ib_wq_flags */1708};1709 1710enum ib_wq_attr_mask {1711	IB_WQ_STATE		= 1 << 0,1712	IB_WQ_CUR_STATE		= 1 << 1,1713	IB_WQ_FLAGS		= 1 << 2,1714};1715 1716struct ib_wq_attr {1717	enum	ib_wq_state	wq_state;1718	enum	ib_wq_state	curr_wq_state;1719	u32			flags; /* Use enum ib_wq_flags */1720	u32			flags_mask; /* Use enum ib_wq_flags */1721};1722 1723struct ib_rwq_ind_table {1724	struct ib_device	*device;1725	struct ib_uobject      *uobject;1726	atomic_t		usecnt;1727	u32		ind_tbl_num;1728	u32		log_ind_tbl_size;1729	struct ib_wq	**ind_tbl;1730};1731 1732struct ib_rwq_ind_table_init_attr {1733	u32		log_ind_tbl_size;1734	/* Each entry is a pointer to Receive Work Queue */1735	struct ib_wq	**ind_tbl;1736};1737 1738enum port_pkey_state {1739	IB_PORT_PKEY_NOT_VALID = 0,1740	IB_PORT_PKEY_VALID = 1,1741	IB_PORT_PKEY_LISTED = 2,1742};1743 1744struct ib_qp_security;1745 1746struct ib_port_pkey {1747	enum port_pkey_state	state;1748	u16			pkey_index;1749	u32			port_num;1750	struct list_head	qp_list;1751	struct list_head	to_error_list;1752	struct ib_qp_security  *sec;1753};1754 1755struct ib_ports_pkeys {1756	struct ib_port_pkey	main;1757	struct ib_port_pkey	alt;1758};1759 1760struct ib_qp_security {1761	struct ib_qp	       *qp;1762	struct ib_device       *dev;1763	/* Hold this mutex when changing port and pkey settings. */1764	struct mutex		mutex;1765	struct ib_ports_pkeys  *ports_pkeys;1766	/* A list of all open shared QP handles.  Required to enforce security1767	 * properly for all users of a shared QP.1768	 */1769	struct list_head        shared_qp_list;1770	void                   *security;1771	bool			destroying;1772	atomic_t		error_list_count;1773	struct completion	error_complete;1774	int			error_comps_pending;1775};1776 1777/*1778 * @max_write_sge: Maximum SGE elements per RDMA WRITE request.1779 * @max_read_sge:  Maximum SGE elements per RDMA READ request.1780 */1781struct ib_qp {1782	struct ib_device       *device;1783	struct ib_pd	       *pd;1784	struct ib_cq	       *send_cq;1785	struct ib_cq	       *recv_cq;1786	spinlock_t		mr_lock;1787	int			mrs_used;1788	struct list_head	rdma_mrs;1789	struct list_head	sig_mrs;1790	struct ib_srq	       *srq;1791	struct completion	srq_completion;1792	struct ib_xrcd	       *xrcd; /* XRC TGT QPs only */1793	struct list_head	xrcd_list;1794 1795	/* count times opened, mcast attaches, flow attaches */1796	atomic_t		usecnt;1797	struct list_head	open_list;1798	struct ib_qp           *real_qp;1799	struct ib_uqp_object   *uobject;1800	void                  (*event_handler)(struct ib_event *, void *);1801	void                  (*registered_event_handler)(struct ib_event *, void *);1802	void		       *qp_context;1803	/* sgid_attrs associated with the AV's */1804	const struct ib_gid_attr *av_sgid_attr;1805	const struct ib_gid_attr *alt_path_sgid_attr;1806	u32			qp_num;1807	u32			max_write_sge;1808	u32			max_read_sge;1809	enum ib_qp_type		qp_type;1810	struct ib_rwq_ind_table *rwq_ind_tbl;1811	struct ib_qp_security  *qp_sec;1812	u32			port;1813 1814	bool			integrity_en;1815	/*1816	 * Implementation details of the RDMA core, don't use in drivers:1817	 */1818	struct rdma_restrack_entry     res;1819 1820	/* The counter the qp is bind to */1821	struct rdma_counter    *counter;1822};1823 1824struct ib_dm {1825	struct ib_device  *device;1826	u32		   length;1827	u32		   flags;1828	struct ib_uobject *uobject;1829	atomic_t	   usecnt;1830};1831 1832struct ib_mr {1833	struct ib_device  *device;1834	struct ib_pd	  *pd;1835	u32		   lkey;1836	u32		   rkey;1837	u64		   iova;1838	u64		   length;1839	unsigned int	   page_size;1840	enum ib_mr_type	   type;1841	bool		   need_inval;1842	union {1843		struct ib_uobject	*uobject;	/* user */1844		struct list_head	qp_entry;	/* FR */1845	};1846 1847	struct ib_dm      *dm;1848	struct ib_sig_attrs *sig_attrs; /* only for IB_MR_TYPE_INTEGRITY MRs */1849	/*1850	 * Implementation details of the RDMA core, don't use in drivers:1851	 */1852	struct rdma_restrack_entry res;1853};1854 1855struct ib_mw {1856	struct ib_device	*device;1857	struct ib_pd		*pd;1858	struct ib_uobject	*uobject;1859	u32			rkey;1860	enum ib_mw_type         type;1861};1862 1863/* Supported steering options */1864enum ib_flow_attr_type {1865	/* steering according to rule specifications */1866	IB_FLOW_ATTR_NORMAL		= 0x0,1867	/* default unicast and multicast rule -1868	 * receive all Eth traffic which isn't steered to any QP1869	 */1870	IB_FLOW_ATTR_ALL_DEFAULT	= 0x1,1871	/* default multicast rule -1872	 * receive all Eth multicast traffic which isn't steered to any QP1873	 */1874	IB_FLOW_ATTR_MC_DEFAULT		= 0x2,1875	/* sniffer rule - receive all port traffic */1876	IB_FLOW_ATTR_SNIFFER		= 0x31877};1878 1879/* Supported steering header types */1880enum ib_flow_spec_type {1881	/* L2 headers*/1882	IB_FLOW_SPEC_ETH		= 0x20,1883	IB_FLOW_SPEC_IB			= 0x22,1884	/* L3 header*/1885	IB_FLOW_SPEC_IPV4		= 0x30,1886	IB_FLOW_SPEC_IPV6		= 0x31,1887	IB_FLOW_SPEC_ESP                = 0x34,1888	/* L4 headers*/1889	IB_FLOW_SPEC_TCP		= 0x40,1890	IB_FLOW_SPEC_UDP		= 0x41,1891	IB_FLOW_SPEC_VXLAN_TUNNEL	= 0x50,1892	IB_FLOW_SPEC_GRE		= 0x51,1893	IB_FLOW_SPEC_MPLS		= 0x60,1894	IB_FLOW_SPEC_INNER		= 0x100,1895	/* Actions */1896	IB_FLOW_SPEC_ACTION_TAG         = 0x1000,1897	IB_FLOW_SPEC_ACTION_DROP        = 0x1001,1898	IB_FLOW_SPEC_ACTION_HANDLE	= 0x1002,1899	IB_FLOW_SPEC_ACTION_COUNT       = 0x1003,1900};1901#define IB_FLOW_SPEC_LAYER_MASK	0xF01902#define IB_FLOW_SPEC_SUPPORT_LAYERS 101903 1904enum ib_flow_flags {1905	IB_FLOW_ATTR_FLAGS_DONT_TRAP = 1UL << 1, /* Continue match, no steal */1906	IB_FLOW_ATTR_FLAGS_EGRESS = 1UL << 2, /* Egress flow */1907	IB_FLOW_ATTR_FLAGS_RESERVED  = 1UL << 3  /* Must be last */1908};1909 1910struct ib_flow_eth_filter {1911	u8	dst_mac[6];1912	u8	src_mac[6];1913	__be16	ether_type;1914	__be16	vlan_tag;1915};1916 1917struct ib_flow_spec_eth {1918	u32			  type;1919	u16			  size;1920	struct ib_flow_eth_filter val;1921	struct ib_flow_eth_filter mask;1922};1923 1924struct ib_flow_ib_filter {1925	__be16 dlid;1926	__u8   sl;1927};1928 1929struct ib_flow_spec_ib {1930	u32			 type;1931	u16			 size;1932	struct ib_flow_ib_filter val;1933	struct ib_flow_ib_filter mask;1934};1935 1936/* IPv4 header flags */1937enum ib_ipv4_flags {1938	IB_IPV4_DONT_FRAG = 0x2, /* Don't enable packet fragmentation */1939	IB_IPV4_MORE_FRAG = 0X4  /* For All fragmented packets except the1940				    last have this flag set */1941};1942 1943struct ib_flow_ipv4_filter {1944	__be32	src_ip;1945	__be32	dst_ip;1946	u8	proto;1947	u8	tos;1948	u8	ttl;1949	u8	flags;1950};1951 1952struct ib_flow_spec_ipv4 {1953	u32			   type;1954	u16			   size;1955	struct ib_flow_ipv4_filter val;1956	struct ib_flow_ipv4_filter mask;1957};1958 1959struct ib_flow_ipv6_filter {1960	u8	src_ip[16];1961	u8	dst_ip[16];1962	__be32	flow_label;1963	u8	next_hdr;1964	u8	traffic_class;1965	u8	hop_limit;1966} __packed;1967 1968struct ib_flow_spec_ipv6 {1969	u32			   type;1970	u16			   size;1971	struct ib_flow_ipv6_filter val;1972	struct ib_flow_ipv6_filter mask;1973};1974 1975struct ib_flow_tcp_udp_filter {1976	__be16	dst_port;1977	__be16	src_port;1978};1979 1980struct ib_flow_spec_tcp_udp {1981	u32			      type;1982	u16			      size;1983	struct ib_flow_tcp_udp_filter val;1984	struct ib_flow_tcp_udp_filter mask;1985};1986 1987struct ib_flow_tunnel_filter {1988	__be32	tunnel_id;1989};1990 1991/* ib_flow_spec_tunnel describes the Vxlan tunnel1992 * the tunnel_id from val has the vni value1993 */1994struct ib_flow_spec_tunnel {1995	u32			      type;1996	u16			      size;1997	struct ib_flow_tunnel_filter  val;1998	struct ib_flow_tunnel_filter  mask;1999};2000 2001struct ib_flow_esp_filter {2002	__be32	spi;2003	__be32  seq;2004};2005 2006struct ib_flow_spec_esp {2007	u32                           type;2008	u16			      size;2009	struct ib_flow_esp_filter     val;2010	struct ib_flow_esp_filter     mask;2011};2012 2013struct ib_flow_gre_filter {2014	__be16 c_ks_res0_ver;2015	__be16 protocol;2016	__be32 key;2017};2018 2019struct ib_flow_spec_gre {2020	u32                           type;2021	u16			      size;2022	struct ib_flow_gre_filter     val;2023	struct ib_flow_gre_filter     mask;2024};2025 2026struct ib_flow_mpls_filter {2027	__be32 tag;2028};2029 2030struct ib_flow_spec_mpls {2031	u32                           type;2032	u16			      size;2033	struct ib_flow_mpls_filter     val;2034	struct ib_flow_mpls_filter     mask;2035};2036 2037struct ib_flow_spec_action_tag {2038	enum ib_flow_spec_type	      type;2039	u16			      size;2040	u32                           tag_id;2041};2042 2043struct ib_flow_spec_action_drop {2044	enum ib_flow_spec_type	      type;2045	u16			      size;2046};2047 2048struct ib_flow_spec_action_handle {2049	enum ib_flow_spec_type	      type;2050	u16			      size;2051	struct ib_flow_action	     *act;2052};2053 2054enum ib_counters_description {2055	IB_COUNTER_PACKETS,2056	IB_COUNTER_BYTES,2057};2058 2059struct ib_flow_spec_action_count {2060	enum ib_flow_spec_type type;2061	u16 size;2062	struct ib_counters *counters;2063};2064 2065union ib_flow_spec {2066	struct {2067		u32			type;2068		u16			size;2069	};2070	struct ib_flow_spec_eth		eth;2071	struct ib_flow_spec_ib		ib;2072	struct ib_flow_spec_ipv4        ipv4;2073	struct ib_flow_spec_tcp_udp	tcp_udp;2074	struct ib_flow_spec_ipv6        ipv6;2075	struct ib_flow_spec_tunnel      tunnel;2076	struct ib_flow_spec_esp		esp;2077	struct ib_flow_spec_gre		gre;2078	struct ib_flow_spec_mpls	mpls;2079	struct ib_flow_spec_action_tag  flow_tag;2080	struct ib_flow_spec_action_drop drop;2081	struct ib_flow_spec_action_handle action;2082	struct ib_flow_spec_action_count flow_count;2083};2084 2085struct ib_flow_attr {2086	enum ib_flow_attr_type type;2087	u16	     size;2088	u16	     priority;2089	u32	     flags;2090	u8	     num_of_specs;2091	u32	     port;2092	union ib_flow_spec flows[];2093};2094 2095struct ib_flow {2096	struct ib_qp		*qp;2097	struct ib_device	*device;2098	struct ib_uobject	*uobject;2099};2100 2101enum ib_flow_action_type {2102	IB_FLOW_ACTION_UNSPECIFIED,2103	IB_FLOW_ACTION_ESP = 1,2104};2105 2106struct ib_flow_action_attrs_esp_keymats {2107	enum ib_uverbs_flow_action_esp_keymat			protocol;2108	union {2109		struct ib_uverbs_flow_action_esp_keymat_aes_gcm aes_gcm;2110	} keymat;2111};2112 2113struct ib_flow_action_attrs_esp_replays {2114	enum ib_uverbs_flow_action_esp_replay			protocol;2115	union {2116		struct ib_uverbs_flow_action_esp_replay_bmp	bmp;2117	} replay;2118};2119 2120enum ib_flow_action_attrs_esp_flags {2121	/* All user-space flags at the top: Use enum ib_uverbs_flow_action_esp_flags2122	 * This is done in order to share the same flags between user-space and2123	 * kernel and spare an unnecessary translation.2124	 */2125 2126	/* Kernel flags */2127	IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED	= 1ULL << 32,2128	IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS	= 1ULL << 33,2129};2130 2131struct ib_flow_spec_list {2132	struct ib_flow_spec_list	*next;2133	union ib_flow_spec		spec;2134};2135 2136struct ib_flow_action_attrs_esp {2137	struct ib_flow_action_attrs_esp_keymats		*keymat;2138	struct ib_flow_action_attrs_esp_replays		*replay;2139	struct ib_flow_spec_list			*encap;2140	/* Used only if IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED is enabled.2141	 * Value of 0 is a valid value.2142	 */2143	u32						esn;2144	u32						spi;2145	u32						seq;2146	u32						tfc_pad;2147	/* Use enum ib_flow_action_attrs_esp_flags */2148	u64						flags;2149	u64						hard_limit_pkts;2150};2151 2152struct ib_flow_action {2153	struct ib_device		*device;2154	struct ib_uobject		*uobject;2155	enum ib_flow_action_type	type;2156	atomic_t			usecnt;2157};2158 2159struct ib_mad;2160 2161enum ib_process_mad_flags {2162	IB_MAD_IGNORE_MKEY	= 1,2163	IB_MAD_IGNORE_BKEY	= 2,2164	IB_MAD_IGNORE_ALL	= IB_MAD_IGNORE_MKEY | IB_MAD_IGNORE_BKEY2165};2166 2167enum ib_mad_result {2168	IB_MAD_RESULT_FAILURE  = 0,      /* (!SUCCESS is the important flag) */2169	IB_MAD_RESULT_SUCCESS  = 1 << 0, /* MAD was successfully processed   */2170	IB_MAD_RESULT_REPLY    = 1 << 1, /* Reply packet needs to be sent    */2171	IB_MAD_RESULT_CONSUMED = 1 << 2  /* Packet consumed: stop processing */2172};2173 2174struct ib_port_cache {2175	u64		      subnet_prefix;2176	struct ib_pkey_cache  *pkey;2177	struct ib_gid_table   *gid;2178	u8                     lmc;2179	enum ib_port_state     port_state;2180};2181 2182struct ib_port_immutable {2183	int                           pkey_tbl_len;2184	int                           gid_tbl_len;2185	u32                           core_cap_flags;2186	u32                           max_mad_size;2187};2188 2189struct ib_port_data {2190	struct ib_device *ib_dev;2191 2192	struct ib_port_immutable immutable;2193 2194	spinlock_t pkey_list_lock;2195 2196	spinlock_t netdev_lock;2197 2198	struct list_head pkey_list;2199 2200	struct ib_port_cache cache;2201 2202	struct net_device __rcu *netdev;2203	netdevice_tracker netdev_tracker;2204	struct hlist_node ndev_hash_link;2205	struct rdma_port_counter port_counter;2206	struct ib_port *sysfs;2207};2208 2209/* rdma netdev type - specifies protocol type */2210enum rdma_netdev_t {2211	RDMA_NETDEV_OPA_VNIC,2212	RDMA_NETDEV_IPOIB,2213};2214 2215/**2216 * struct rdma_netdev - rdma netdev2217 * For cases where netstack interfacing is required.2218 */2219struct rdma_netdev {2220	void              *clnt_priv;2221	struct ib_device  *hca;2222	u32		   port_num;2223	int                mtu;2224 2225	/*2226	 * cleanup function must be specified.2227	 * FIXME: This is only used for OPA_VNIC and that usage should be2228	 * removed too.2229	 */2230	void (*free_rdma_netdev)(struct net_device *netdev);2231 2232	/* control functions */2233	void (*set_id)(struct net_device *netdev, int id);2234	/* send packet */2235	int (*send)(struct net_device *dev, struct sk_buff *skb,2236		    struct ib_ah *address, u32 dqpn);2237	/* multicast */2238	int (*attach_mcast)(struct net_device *dev, struct ib_device *hca,2239			    union ib_gid *gid, u16 mlid,2240			    int set_qkey, u32 qkey);2241	int (*detach_mcast)(struct net_device *dev, struct ib_device *hca,2242			    union ib_gid *gid, u16 mlid);2243	/* timeout */2244	void (*tx_timeout)(struct net_device *dev, unsigned int txqueue);2245};2246 2247struct rdma_netdev_alloc_params {2248	size_t sizeof_priv;2249	unsigned int txqs;2250	unsigned int rxqs;2251	void *param;2252 2253	int (*initialize_rdma_netdev)(struct ib_device *device, u32 port_num,2254				      struct net_device *netdev, void *param);2255};2256 2257struct ib_odp_counters {2258	atomic64_t faults;2259	atomic64_t invalidations;2260	atomic64_t prefetch;2261};2262 2263struct ib_counters {2264	struct ib_device	*device;2265	struct ib_uobject	*uobject;2266	/* num of objects attached */2267	atomic_t	usecnt;2268};2269 2270struct ib_counters_read_attr {2271	u64	*counters_buff;2272	u32	ncounters;2273	u32	flags; /* use enum ib_read_counters_flags */2274};2275 2276struct uverbs_attr_bundle;2277struct iw_cm_id;2278struct iw_cm_conn_param;2279 2280#define INIT_RDMA_OBJ_SIZE(ib_struct, drv_struct, member)                      \2281	.size_##ib_struct =                                                    \2282		(sizeof(struct drv_struct) +                                   \2283		 BUILD_BUG_ON_ZERO(offsetof(struct drv_struct, member)) +      \2284		 BUILD_BUG_ON_ZERO(                                            \2285			 !__same_type(((struct drv_struct *)NULL)->member,     \2286				      struct ib_struct)))2287 2288#define rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, gfp)                          \2289	((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \2290					   gfp, false))2291 2292#define rdma_zalloc_drv_obj_numa(ib_dev, ib_type)                              \2293	((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \2294					   GFP_KERNEL, true))2295 2296#define rdma_zalloc_drv_obj(ib_dev, ib_type)                                   \2297	rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, GFP_KERNEL)2298 2299#define DECLARE_RDMA_OBJ_SIZE(ib_struct) size_t size_##ib_struct2300 2301struct rdma_user_mmap_entry {2302	struct kref ref;2303	struct ib_ucontext *ucontext;2304	unsigned long start_pgoff;2305	size_t npages;2306	bool driver_removed;2307};2308 2309/* Return the offset (in bytes) the user should pass to libc's mmap() */2310static inline u642311rdma_user_mmap_get_offset(const struct rdma_user_mmap_entry *entry)2312{2313	return (u64)entry->start_pgoff << PAGE_SHIFT;2314}2315 2316/**2317 * struct ib_device_ops - InfiniBand device operations2318 * This structure defines all the InfiniBand device operations, providers will2319 * need to define the supported operations, otherwise they will be set to null.2320 */2321struct ib_device_ops {2322	struct module *owner;2323	enum rdma_driver_id driver_id;2324	u32 uverbs_abi_ver;2325	unsigned int uverbs_no_driver_id_binding:1;2326 2327	/*2328	 * NOTE: New drivers should not make use of device_group; instead new2329	 * device parameter should be exposed via netlink command. This2330	 * mechanism exists only for existing drivers.2331	 */2332	const struct attribute_group *device_group;2333	const struct attribute_group **port_groups;2334 2335	int (*post_send)(struct ib_qp *qp, const struct ib_send_wr *send_wr,2336			 const struct ib_send_wr **bad_send_wr);2337	int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,2338			 const struct ib_recv_wr **bad_recv_wr);2339	void (*drain_rq)(struct ib_qp *qp);2340	void (*drain_sq)(struct ib_qp *qp);2341	int (*poll_cq)(struct ib_cq *cq, int num_entries, struct ib_wc *wc);2342	int (*peek_cq)(struct ib_cq *cq, int wc_cnt);2343	int (*req_notify_cq)(struct ib_cq *cq, enum ib_cq_notify_flags flags);2344	int (*post_srq_recv)(struct ib_srq *srq,2345			     const struct ib_recv_wr *recv_wr,2346			     const struct ib_recv_wr **bad_recv_wr);2347	int (*process_mad)(struct ib_device *device, int process_mad_flags,2348			   u32 port_num, const struct ib_wc *in_wc,2349			   const struct ib_grh *in_grh,2350			   const struct ib_mad *in_mad, struct ib_mad *out_mad,2351			   size_t *out_mad_size, u16 *out_mad_pkey_index);2352	int (*query_device)(struct ib_device *device,2353			    struct ib_device_attr *device_attr,2354			    struct ib_udata *udata);2355	int (*modify_device)(struct ib_device *device, int device_modify_mask,2356			     struct ib_device_modify *device_modify);2357	void (*get_dev_fw_str)(struct ib_device *device, char *str);2358	const struct cpumask *(*get_vector_affinity)(struct ib_device *ibdev,2359						     int comp_vector);2360	int (*query_port)(struct ib_device *device, u32 port_num,2361			  struct ib_port_attr *port_attr);2362	int (*modify_port)(struct ib_device *device, u32 port_num,2363			   int port_modify_mask,2364			   struct ib_port_modify *port_modify);2365	/**2366	 * The following mandatory functions are used only at device2367	 * registration.  Keep functions such as these at the end of this2368	 * structure to avoid cache line misses when accessing struct ib_device2369	 * in fast paths.2370	 */2371	int (*get_port_immutable)(struct ib_device *device, u32 port_num,2372				  struct ib_port_immutable *immutable);2373	enum rdma_link_layer (*get_link_layer)(struct ib_device *device,2374					       u32 port_num);2375	/**2376	 * When calling get_netdev, the HW vendor's driver should return the2377	 * net device of device @device at port @port_num or NULL if such2378	 * a net device doesn't exist. The vendor driver should call dev_hold2379	 * on this net device. The HW vendor's device driver must guarantee2380	 * that this function returns NULL before the net device has finished2381	 * NETDEV_UNREGISTER state.2382	 */2383	struct net_device *(*get_netdev)(struct ib_device *device,2384					 u32 port_num);2385	/**2386	 * rdma netdev operation2387	 *2388	 * Driver implementing alloc_rdma_netdev or rdma_netdev_get_params2389	 * must return -EOPNOTSUPP if it doesn't support the specified type.2390	 */2391	struct net_device *(*alloc_rdma_netdev)(2392		struct ib_device *device, u32 port_num, enum rdma_netdev_t type,2393		const char *name, unsigned char name_assign_type,2394		void (*setup)(struct net_device *));2395 2396	int (*rdma_netdev_get_params)(struct ib_device *device, u32 port_num,2397				      enum rdma_netdev_t type,2398				      struct rdma_netdev_alloc_params *params);2399	/**2400	 * query_gid should be return GID value for @device, when @port_num2401	 * link layer is either IB or iWarp. It is no-op if @port_num port2402	 * is RoCE link layer.2403	 */2404	int (*query_gid)(struct ib_device *device, u32 port_num, int index,2405			 union ib_gid *gid);2406	/**2407	 * When calling add_gid, the HW vendor's driver should add the gid2408	 * of device of port at gid index available at @attr. Meta-info of2409	 * that gid (for example, the network device related to this gid) is2410	 * available at @attr. @context allows the HW vendor driver to store2411	 * extra information together with a GID entry. The HW vendor driver may2412	 * allocate memory to contain this information and store it in @context2413	 * when a new GID entry is written to. Params are consistent until the2414	 * next call of add_gid or delete_gid. The function should return 0 on2415	 * success or error otherwise. The function could be called2416	 * concurrently for different ports. This function is only called when2417	 * roce_gid_table is used.2418	 */2419	int (*add_gid)(const struct ib_gid_attr *attr, void **context);2420	/**2421	 * When calling del_gid, the HW vendor's driver should delete the2422	 * gid of device @device at gid index gid_index of port port_num2423	 * available in @attr.2424	 * Upon the deletion of a GID entry, the HW vendor must free any2425	 * allocated memory. The caller will clear @context afterwards.2426	 * This function is only called when roce_gid_table is used.2427	 */2428	int (*del_gid)(const struct ib_gid_attr *attr, void **context);2429	int (*query_pkey)(struct ib_device *device, u32 port_num, u16 index,2430			  u16 *pkey);2431	int (*alloc_ucontext)(struct ib_ucontext *context,2432			      struct ib_udata *udata);2433	void (*dealloc_ucontext)(struct ib_ucontext *context);2434	int (*mmap)(struct ib_ucontext *context, struct vm_area_struct *vma);2435	/**2436	 * This will be called once refcount of an entry in mmap_xa reaches2437	 * zero. The type of the memory that was mapped may differ between2438	 * entries and is opaque to the rdma_user_mmap interface.2439	 * Therefore needs to be implemented by the driver in mmap_free.2440	 */2441	void (*mmap_free)(struct rdma_user_mmap_entry *entry);2442	void (*disassociate_ucontext)(struct ib_ucontext *ibcontext);2443	int (*alloc_pd)(struct ib_pd *pd, struct ib_udata *udata);2444	int (*dealloc_pd)(struct ib_pd *pd, struct ib_udata *udata);2445	int (*create_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr,2446			 struct ib_udata *udata);2447	int (*create_user_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr,2448			      struct ib_udata *udata);2449	int (*modify_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);2450	int (*query_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);2451	int (*destroy_ah)(struct ib_ah *ah, u32 flags);2452	int (*create_srq)(struct ib_srq *srq,2453			  struct ib_srq_init_attr *srq_init_attr,2454			  struct ib_udata *udata);2455	int (*modify_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr,2456			  enum ib_srq_attr_mask srq_attr_mask,2457			  struct ib_udata *udata);2458	int (*query_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr);2459	int (*destroy_srq)(struct ib_srq *srq, struct ib_udata *udata);2460	int (*create_qp)(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr,2461			 struct ib_udata *udata);2462	int (*modify_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,2463			 int qp_attr_mask, struct ib_udata *udata);2464	int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,2465			int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);2466	int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata);2467	int (*create_cq)(struct ib_cq *cq, const struct ib_cq_init_attr *attr,2468			 struct uverbs_attr_bundle *attrs);2469	int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);2470	int (*destroy_cq)(struct ib_cq *cq, struct ib_udata *udata);2471	int (*resize_cq)(struct ib_cq *cq, int cqe, struct ib_udata *udata);2472	struct ib_mr *(*get_dma_mr)(struct ib_pd *pd, int mr_access_flags);2473	struct ib_mr *(*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length,2474				     u64 virt_addr, int mr_access_flags,2475				     struct ib_udata *udata);2476	struct ib_mr *(*reg_user_mr_dmabuf)(struct ib_pd *pd, u64 offset,2477					    u64 length, u64 virt_addr, int fd,2478					    int mr_access_flags,2479					    struct uverbs_attr_bundle *attrs);2480	struct ib_mr *(*rereg_user_mr)(struct ib_mr *mr, int flags, u64 start,2481				       u64 length, u64 virt_addr,2482				       int mr_access_flags, struct ib_pd *pd,2483				       struct ib_udata *udata);2484	int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata);2485	struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type,2486				  u32 max_num_sg);2487	struct ib_mr *(*alloc_mr_integrity)(struct ib_pd *pd,2488					    u32 max_num_data_sg,2489					    u32 max_num_meta_sg);2490	int (*advise_mr)(struct ib_pd *pd,2491			 enum ib_uverbs_advise_mr_advice advice, u32 flags,2492			 struct ib_sge *sg_list, u32 num_sge,2493			 struct uverbs_attr_bundle *attrs);2494 2495	/*2496	 * Kernel users should universally support relaxed ordering (RO), as2497	 * they are designed to read data only after observing the CQE and use2498	 * the DMA API correctly.2499	 *2500	 * Some drivers implicitly enable RO if platform supports it.2501	 */2502	int (*map_mr_sg)(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,2503			 unsigned int *sg_offset);2504	int (*check_mr_status)(struct ib_mr *mr, u32 check_mask,2505			       struct ib_mr_status *mr_status);2506	int (*alloc_mw)(struct ib_mw *mw, struct ib_udata *udata);2507	int (*dealloc_mw)(struct ib_mw *mw);2508	int (*attach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);2509	int (*detach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);2510	int (*alloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);2511	int (*dealloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);2512	struct ib_flow *(*create_flow)(struct ib_qp *qp,2513				       struct ib_flow_attr *flow_attr,2514				       struct ib_udata *udata);2515	int (*destroy_flow)(struct ib_flow *flow_id);2516	int (*destroy_flow_action)(struct ib_flow_action *action);2517	int (*set_vf_link_state)(struct ib_device *device, int vf, u32 port,2518				 int state);2519	int (*get_vf_config)(struct ib_device *device, int vf, u32 port,2520			     struct ifla_vf_info *ivf);2521	int (*get_vf_stats)(struct ib_device *device, int vf, u32 port,2522			    struct ifla_vf_stats *stats);2523	int (*get_vf_guid)(struct ib_device *device, int vf, u32 port,2524			    struct ifla_vf_guid *node_guid,2525			    struct ifla_vf_guid *port_guid);2526	int (*set_vf_guid)(struct ib_device *device, int vf, u32 port, u64 guid,2527			   int type);2528	struct ib_wq *(*create_wq)(struct ib_pd *pd,2529				   struct ib_wq_init_attr *init_attr,2530				   struct ib_udata *udata);2531	int (*destroy_wq)(struct ib_wq *wq, struct ib_udata *udata);2532	int (*modify_wq)(struct ib_wq *wq, struct ib_wq_attr *attr,2533			 u32 wq_attr_mask, struct ib_udata *udata);2534	int (*create_rwq_ind_table)(struct ib_rwq_ind_table *ib_rwq_ind_table,2535				    struct ib_rwq_ind_table_init_attr *init_attr,2536				    struct ib_udata *udata);2537	int (*destroy_rwq_ind_table)(struct ib_rwq_ind_table *wq_ind_table);2538	struct ib_dm *(*alloc_dm)(struct ib_device *device,2539				  struct ib_ucontext *context,2540				  struct ib_dm_alloc_attr *attr,2541				  struct uverbs_attr_bundle *attrs);2542	int (*dealloc_dm)(struct ib_dm *dm, struct uverbs_attr_bundle *attrs);2543	struct ib_mr *(*reg_dm_mr)(struct ib_pd *pd, struct ib_dm *dm,2544				   struct ib_dm_mr_attr *attr,2545				   struct uverbs_attr_bundle *attrs);2546	int (*create_counters)(struct ib_counters *counters,2547			       struct uverbs_attr_bundle *attrs);2548	int (*destroy_counters)(struct ib_counters *counters);2549	int (*read_counters)(struct ib_counters *counters,2550			     struct ib_counters_read_attr *counters_read_attr,2551			     struct uverbs_attr_bundle *attrs);2552	int (*map_mr_sg_pi)(struct ib_mr *mr, struct scatterlist *data_sg,2553			    int data_sg_nents, unsigned int *data_sg_offset,2554			    struct scatterlist *meta_sg, int meta_sg_nents,2555			    unsigned int *meta_sg_offset);2556 2557	/**2558	 * alloc_hw_[device,port]_stats - Allocate a struct rdma_hw_stats and2559	 *   fill in the driver initialized data.  The struct is kfree()'ed by2560	 *   the sysfs core when the device is removed.  A lifespan of -1 in the2561	 *   return struct tells the core to set a default lifespan.2562	 */2563	struct rdma_hw_stats *(*alloc_hw_device_stats)(struct ib_device *device);2564	struct rdma_hw_stats *(*alloc_hw_port_stats)(struct ib_device *device,2565						     u32 port_num);2566	/**2567	 * get_hw_stats - Fill in the counter value(s) in the stats struct.2568	 * @index - The index in the value array we wish to have updated, or2569	 *   num_counters if we want all stats updated2570	 * Return codes -2571	 *   < 0 - Error, no counters updated2572	 *   index - Updated the single counter pointed to by index2573	 *   num_counters - Updated all counters (will reset the timestamp2574	 *     and prevent further calls for lifespan milliseconds)2575	 * Drivers are allowed to update all counters in leiu of just the2576	 *   one given in index at their option2577	 */2578	int (*get_hw_stats)(struct ib_device *device,2579			    struct rdma_hw_stats *stats, u32 port, int index);2580 2581	/**2582	 * modify_hw_stat - Modify the counter configuration2583	 * @enable: true/false when enable/disable a counter2584	 * Return codes - 0 on success or error code otherwise.2585	 */2586	int (*modify_hw_stat)(struct ib_device *device, u32 port,2587			      unsigned int counter_index, bool enable);2588	/**2589	 * Allows rdma drivers to add their own restrack attributes.2590	 */2591	int (*fill_res_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);2592	int (*fill_res_mr_entry_raw)(struct sk_buff *msg, struct ib_mr *ibmr);2593	int (*fill_res_cq_entry)(struct sk_buff *msg, struct ib_cq *ibcq);2594	int (*fill_res_cq_entry_raw)(struct sk_buff *msg, struct ib_cq *ibcq);2595	int (*fill_res_qp_entry)(struct sk_buff *msg, struct ib_qp *ibqp);2596	int (*fill_res_qp_entry_raw)(struct sk_buff *msg, struct ib_qp *ibqp);2597	int (*fill_res_cm_id_entry)(struct sk_buff *msg, struct rdma_cm_id *id);2598	int (*fill_res_srq_entry)(struct sk_buff *msg, struct ib_srq *ib_srq);2599	int (*fill_res_srq_entry_raw)(struct sk_buff *msg, struct ib_srq *ib_srq);2600 2601	/* Device lifecycle callbacks */2602	/*2603	 * Called after the device becomes registered, before clients are2604	 * attached2605	 */2606	int (*enable_driver)(struct ib_device *dev);2607	/*2608	 * This is called as part of ib_dealloc_device().2609	 */2610	void (*dealloc_driver)(struct ib_device *dev);2611 2612	/* iWarp CM callbacks */2613	void (*iw_add_ref)(struct ib_qp *qp);2614	void (*iw_rem_ref)(struct ib_qp *qp);2615	struct ib_qp *(*iw_get_qp)(struct ib_device *device, int qpn);2616	int (*iw_connect)(struct iw_cm_id *cm_id,2617			  struct iw_cm_conn_param *conn_param);2618	int (*iw_accept)(struct iw_cm_id *cm_id,2619			 struct iw_cm_conn_param *conn_param);2620	int (*iw_reject)(struct iw_cm_id *cm_id, const void *pdata,2621			 u8 pdata_len);2622	int (*iw_create_listen)(struct iw_cm_id *cm_id, int backlog);2623	int (*iw_destroy_listen)(struct iw_cm_id *cm_id);2624	/**2625	 * counter_bind_qp - Bind a QP to a counter.2626	 * @counter - The counter to be bound. If counter->id is zero then2627	 *   the driver needs to allocate a new counter and set counter->id2628	 */2629	int (*counter_bind_qp)(struct rdma_counter *counter, struct ib_qp *qp);2630	/**2631	 * counter_unbind_qp - Unbind the qp from the dynamically-allocated2632	 *   counter and bind it onto the default one2633	 */2634	int (*counter_unbind_qp)(struct ib_qp *qp);2635	/**2636	 * counter_dealloc -De-allocate the hw counter2637	 */2638	int (*counter_dealloc)(struct rdma_counter *counter);2639	/**2640	 * counter_alloc_stats - Allocate a struct rdma_hw_stats and fill in2641	 * the driver initialized data.2642	 */2643	struct rdma_hw_stats *(*counter_alloc_stats)(2644		struct rdma_counter *counter);2645	/**2646	 * counter_update_stats - Query the stats value of this counter2647	 */2648	int (*counter_update_stats)(struct rdma_counter *counter);2649 2650	/**2651	 * Allows rdma drivers to add their own restrack attributes2652	 * dumped via 'rdma stat' iproute2 command.2653	 */2654	int (*fill_stat_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);2655 2656	/* query driver for its ucontext properties */2657	int (*query_ucontext)(struct ib_ucontext *context,2658			      struct uverbs_attr_bundle *attrs);2659 2660	/*2661	 * Provide NUMA node. This API exists for rdmavt/hfi1 only.2662	 * Everyone else relies on Linux memory management model.2663	 */2664	int (*get_numa_node)(struct ib_device *dev);2665 2666	/**2667	 * add_sub_dev - Add a sub IB device2668	 */2669	struct ib_device *(*add_sub_dev)(struct ib_device *parent,2670					 enum rdma_nl_dev_type type,2671					 const char *name);2672 2673	/**2674	 * del_sub_dev - Delete a sub IB device2675	 */2676	void (*del_sub_dev)(struct ib_device *sub_dev);2677 2678	DECLARE_RDMA_OBJ_SIZE(ib_ah);2679	DECLARE_RDMA_OBJ_SIZE(ib_counters);2680	DECLARE_RDMA_OBJ_SIZE(ib_cq);2681	DECLARE_RDMA_OBJ_SIZE(ib_mw);2682	DECLARE_RDMA_OBJ_SIZE(ib_pd);2683	DECLARE_RDMA_OBJ_SIZE(ib_qp);2684	DECLARE_RDMA_OBJ_SIZE(ib_rwq_ind_table);2685	DECLARE_RDMA_OBJ_SIZE(ib_srq);2686	DECLARE_RDMA_OBJ_SIZE(ib_ucontext);2687	DECLARE_RDMA_OBJ_SIZE(ib_xrcd);2688};2689 2690struct ib_core_device {2691	/* device must be the first element in structure until,2692	 * union of ib_core_device and device exists in ib_device.2693	 */2694	struct device dev;2695	possible_net_t rdma_net;2696	struct kobject *ports_kobj;2697	struct list_head port_list;2698	struct ib_device *owner; /* reach back to owner ib_device */2699};2700 2701struct rdma_restrack_root;2702struct ib_device {2703	/* Do not access @dma_device directly from ULP nor from HW drivers. */2704	struct device                *dma_device;2705	struct ib_device_ops	     ops;2706	char                          name[IB_DEVICE_NAME_MAX];2707	struct rcu_head rcu_head;2708 2709	struct list_head              event_handler_list;2710	/* Protects event_handler_list */2711	struct rw_semaphore event_handler_rwsem;2712 2713	/* Protects QP's event_handler calls and open_qp list */2714	spinlock_t qp_open_list_lock;2715 2716	struct rw_semaphore	      client_data_rwsem;2717	struct xarray                 client_data;2718	struct mutex                  unregistration_lock;2719 2720	/* Synchronize GID, Pkey cache entries, subnet prefix, LMC */2721	rwlock_t cache_lock;2722	/**2723	 * port_data is indexed by port number2724	 */2725	struct ib_port_data *port_data;2726 2727	int			      num_comp_vectors;2728 2729	union {2730		struct device		dev;2731		struct ib_core_device	coredev;2732	};2733 2734	/* First group is for device attributes,2735	 * Second group is for driver provided attributes (optional).2736	 * Third group is for the hw_stats2737	 * It is a NULL terminated array.2738	 */2739	const struct attribute_group	*groups[4];2740 2741	u64			     uverbs_cmd_mask;2742 2743	char			     node_desc[IB_DEVICE_NODE_DESC_MAX];2744	__be64			     node_guid;2745	u32			     local_dma_lkey;2746	u16                          is_switch:1;2747	/* Indicates kernel verbs support, should not be used in drivers */2748	u16                          kverbs_provider:1;2749	/* CQ adaptive moderation (RDMA DIM) */2750	u16                          use_cq_dim:1;2751	u8                           node_type;2752	u32			     phys_port_cnt;2753	struct ib_device_attr        attrs;2754	struct hw_stats_device_data *hw_stats_data;2755 2756#ifdef CONFIG_CGROUP_RDMA2757	struct rdmacg_device         cg_device;2758#endif2759 2760	u32                          index;2761 2762	spinlock_t                   cq_pools_lock;2763	struct list_head             cq_pools[IB_POLL_LAST_POOL_TYPE + 1];2764 2765	struct rdma_restrack_root *res;2766 2767	const struct uapi_definition   *driver_def;2768 2769	/*2770	 * Positive refcount indicates that the device is currently2771	 * registered and cannot be unregistered.2772	 */2773	refcount_t refcount;2774	struct completion unreg_completion;2775	struct work_struct unregistration_work;2776 2777	const struct rdma_link_ops *link_ops;2778 2779	/* Protects compat_devs xarray modifications */2780	struct mutex compat_devs_mutex;2781	/* Maintains compat devices for each net namespace */2782	struct xarray compat_devs;2783 2784	/* Used by iWarp CM */2785	char iw_ifname[IFNAMSIZ];2786	u32 iw_driver_flags;2787	u32 lag_flags;2788 2789	/* A parent device has a list of sub-devices */2790	struct mutex subdev_lock;2791	struct list_head subdev_list_head;2792 2793	/* A sub device has a type and a parent */2794	enum rdma_nl_dev_type type;2795	struct ib_device *parent;2796	struct list_head subdev_list;2797 2798	enum rdma_nl_name_assign_type name_assign_type;2799};2800 2801static inline void *rdma_zalloc_obj(struct ib_device *dev, size_t size,2802				    gfp_t gfp, bool is_numa_aware)2803{2804	if (is_numa_aware && dev->ops.get_numa_node)2805		return kzalloc_node(size, gfp, dev->ops.get_numa_node(dev));2806 2807	return kzalloc(size, gfp);2808}2809 2810struct ib_client_nl_info;2811struct ib_client {2812	const char *name;2813	int (*add)(struct ib_device *ibdev);2814	void (*remove)(struct ib_device *, void *client_data);2815	void (*rename)(struct ib_device *dev, void *client_data);2816	int (*get_nl_info)(struct ib_device *ibdev, void *client_data,2817			   struct ib_client_nl_info *res);2818	int (*get_global_nl_info)(struct ib_client_nl_info *res);2819 2820	/* Returns the net_dev belonging to this ib_client and matching the2821	 * given parameters.2822	 * @dev:	 An RDMA device that the net_dev use for communication.2823	 * @port:	 A physical port number on the RDMA device.2824	 * @pkey:	 P_Key that the net_dev uses if applicable.2825	 * @gid:	 A GID that the net_dev uses to communicate.2826	 * @addr:	 An IP address the net_dev is configured with.2827	 * @client_data: The device's client data set by ib_set_client_data().2828	 *2829	 * An ib_client that implements a net_dev on top of RDMA devices2830	 * (such as IP over IB) should implement this callback, allowing the2831	 * rdma_cm module to find the right net_dev for a given request.2832	 *2833	 * The caller is responsible for calling dev_put on the returned2834	 * netdev. */2835	struct net_device *(*get_net_dev_by_params)(2836			struct ib_device *dev,2837			u32 port,2838			u16 pkey,2839			const union ib_gid *gid,2840			const struct sockaddr *addr,2841			void *client_data);2842 2843	refcount_t uses;2844	struct completion uses_zero;2845	u32 client_id;2846 2847	/* kverbs are not required by the client */2848	u8 no_kverbs_req:1;2849};2850 2851/*2852 * IB block DMA iterator2853 *2854 * Iterates the DMA-mapped SGL in contiguous memory blocks aligned2855 * to a HW supported page size.2856 */2857struct ib_block_iter {2858	/* internal states */2859	struct scatterlist *__sg;	/* sg holding the current aligned block */2860	dma_addr_t __dma_addr;		/* unaligned DMA address of this block */2861	size_t __sg_numblocks;		/* ib_umem_num_dma_blocks() */2862	unsigned int __sg_nents;	/* number of SG entries */2863	unsigned int __sg_advance;	/* number of bytes to advance in sg in next step */2864	unsigned int __pg_bit;		/* alignment of current block */2865};2866 2867struct ib_device *_ib_alloc_device(size_t size);2868#define ib_alloc_device(drv_struct, member)                                    \2869	container_of(_ib_alloc_device(sizeof(struct drv_struct) +              \2870				      BUILD_BUG_ON_ZERO(offsetof(              \2871					      struct drv_struct, member))),    \2872		     struct drv_struct, member)2873 2874void ib_dealloc_device(struct ib_device *device);2875 2876void ib_get_device_fw_str(struct ib_device *device, char *str);2877 2878int ib_register_device(struct ib_device *device, const char *name,2879		       struct device *dma_device);2880void ib_unregister_device(struct ib_device *device);2881void ib_unregister_driver(enum rdma_driver_id driver_id);2882void ib_unregister_device_and_put(struct ib_device *device);2883void ib_unregister_device_queued(struct ib_device *ib_dev);2884 2885int ib_register_client   (struct ib_client *client);2886void ib_unregister_client(struct ib_client *client);2887 2888void __rdma_block_iter_start(struct ib_block_iter *biter,2889			     struct scatterlist *sglist,2890			     unsigned int nents,2891			     unsigned long pgsz);2892bool __rdma_block_iter_next(struct ib_block_iter *biter);2893 2894/**2895 * rdma_block_iter_dma_address - get the aligned dma address of the current2896 * block held by the block iterator.2897 * @biter: block iterator holding the memory block2898 */2899static inline dma_addr_t2900rdma_block_iter_dma_address(struct ib_block_iter *biter)2901{2902	return biter->__dma_addr & ~(BIT_ULL(biter->__pg_bit) - 1);2903}2904 2905/**2906 * rdma_for_each_block - iterate over contiguous memory blocks of the sg list2907 * @sglist: sglist to iterate over2908 * @biter: block iterator holding the memory block2909 * @nents: maximum number of sg entries to iterate over2910 * @pgsz: best HW supported page size to use2911 *2912 * Callers may use rdma_block_iter_dma_address() to get each2913 * blocks aligned DMA address.2914 */2915#define rdma_for_each_block(sglist, biter, nents, pgsz)		\2916	for (__rdma_block_iter_start(biter, sglist, nents,	\2917				     pgsz);			\2918	     __rdma_block_iter_next(biter);)2919 2920/**2921 * ib_get_client_data - Get IB client context2922 * @device:Device to get context for2923 * @client:Client to get context for2924 *2925 * ib_get_client_data() returns the client context data set with2926 * ib_set_client_data(). This can only be called while the client is2927 * registered to the device, once the ib_client remove() callback returns this2928 * cannot be called.2929 */2930static inline void *ib_get_client_data(struct ib_device *device,2931				       struct ib_client *client)2932{2933	return xa_load(&device->client_data, client->client_id);2934}2935void  ib_set_client_data(struct ib_device *device, struct ib_client *client,2936			 void *data);2937void ib_set_device_ops(struct ib_device *device,2938		       const struct ib_device_ops *ops);2939 2940int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,2941		      unsigned long pfn, unsigned long size, pgprot_t prot,2942		      struct rdma_user_mmap_entry *entry);2943int rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext,2944				struct rdma_user_mmap_entry *entry,2945				size_t length);2946int rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext,2947				      struct rdma_user_mmap_entry *entry,2948				      size_t length, u32 min_pgoff,2949				      u32 max_pgoff);2950 2951static inline int2952rdma_user_mmap_entry_insert_exact(struct ib_ucontext *ucontext,2953				  struct rdma_user_mmap_entry *entry,2954				  size_t length, u32 pgoff)2955{2956	return rdma_user_mmap_entry_insert_range(ucontext, entry, length, pgoff,2957						 pgoff);2958}2959 2960struct rdma_user_mmap_entry *2961rdma_user_mmap_entry_get_pgoff(struct ib_ucontext *ucontext,2962			       unsigned long pgoff);2963struct rdma_user_mmap_entry *2964rdma_user_mmap_entry_get(struct ib_ucontext *ucontext,2965			 struct vm_area_struct *vma);2966void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry);2967 2968void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry);2969 2970static inline int ib_copy_from_udata(void *dest, struct ib_udata *udata, size_t len)2971{2972	return copy_from_user(dest, udata->inbuf, len) ? -EFAULT : 0;2973}2974 2975static inline int ib_copy_to_udata(struct ib_udata *udata, void *src, size_t len)2976{2977	return copy_to_user(udata->outbuf, src, len) ? -EFAULT : 0;2978}2979 2980static inline bool ib_is_buffer_cleared(const void __user *p,2981					size_t len)2982{2983	bool ret;2984	u8 *buf;2985 2986	if (len > USHRT_MAX)2987		return false;2988 2989	buf = memdup_user(p, len);2990	if (IS_ERR(buf))2991		return false;2992 2993	ret = !memchr_inv(buf, 0, len);2994	kfree(buf);2995	return ret;2996}2997 2998static inline bool ib_is_udata_cleared(struct ib_udata *udata,2999				       size_t offset,3000				       size_t len)3001{3002	return ib_is_buffer_cleared(udata->inbuf + offset, len);3003}3004 3005/**3006 * ib_modify_qp_is_ok - Check that the supplied attribute mask3007 * contains all required attributes and no attributes not allowed for3008 * the given QP state transition.3009 * @cur_state: Current QP state3010 * @next_state: Next QP state3011 * @type: QP type3012 * @mask: Mask of supplied QP attributes3013 *3014 * This function is a helper function that a low-level driver's3015 * modify_qp method can use to validate the consumer's input.  It3016 * checks that cur_state and next_state are valid QP states, that a3017 * transition from cur_state to next_state is allowed by the IB spec,3018 * and that the attribute mask supplied is allowed for the transition.3019 */3020bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,3021			enum ib_qp_type type, enum ib_qp_attr_mask mask);3022 3023void ib_register_event_handler(struct ib_event_handler *event_handler);3024void ib_unregister_event_handler(struct ib_event_handler *event_handler);3025void ib_dispatch_event(const struct ib_event *event);3026 3027int ib_query_port(struct ib_device *device,3028		  u32 port_num, struct ib_port_attr *port_attr);3029 3030enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device,3031					       u32 port_num);3032 3033/**3034 * rdma_cap_ib_switch - Check if the device is IB switch3035 * @device: Device to check3036 *3037 * Device driver is responsible for setting is_switch bit on3038 * in ib_device structure at init time.3039 *3040 * Return: true if the device is IB switch.3041 */3042static inline bool rdma_cap_ib_switch(const struct ib_device *device)3043{3044	return device->is_switch;3045}3046 3047/**3048 * rdma_start_port - Return the first valid port number for the device3049 * specified3050 *3051 * @device: Device to be checked3052 *3053 * Return start port number3054 */3055static inline u32 rdma_start_port(const struct ib_device *device)3056{3057	return rdma_cap_ib_switch(device) ? 0 : 1;3058}3059 3060/**3061 * rdma_for_each_port - Iterate over all valid port numbers of the IB device3062 * @device - The struct ib_device * to iterate over3063 * @iter - The unsigned int to store the port number3064 */3065#define rdma_for_each_port(device, iter)                                       \3066	for (iter = rdma_start_port(device +				       \3067				    BUILD_BUG_ON_ZERO(!__same_type(u32,	       \3068								   iter)));    \3069	     iter <= rdma_end_port(device); iter++)3070 3071/**3072 * rdma_end_port - Return the last valid port number for the device3073 * specified3074 *3075 * @device: Device to be checked3076 *3077 * Return last port number3078 */3079static inline u32 rdma_end_port(const struct ib_device *device)3080{3081	return rdma_cap_ib_switch(device) ? 0 : device->phys_port_cnt;3082}3083 3084static inline int rdma_is_port_valid(const struct ib_device *device,3085				     unsigned int port)3086{3087	return (port >= rdma_start_port(device) &&3088		port <= rdma_end_port(device));3089}3090 3091static inline bool rdma_is_grh_required(const struct ib_device *device,3092					u32 port_num)3093{3094	return device->port_data[port_num].immutable.core_cap_flags &3095	       RDMA_CORE_PORT_IB_GRH_REQUIRED;3096}3097 3098static inline bool rdma_protocol_ib(const struct ib_device *device,3099				    u32 port_num)3100{3101	return device->port_data[port_num].immutable.core_cap_flags &3102	       RDMA_CORE_CAP_PROT_IB;3103}3104 3105static inline bool rdma_protocol_roce(const struct ib_device *device,3106				      u32 port_num)3107{3108	return device->port_data[port_num].immutable.core_cap_flags &3109	       (RDMA_CORE_CAP_PROT_ROCE | RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP);3110}3111 3112static inline bool rdma_protocol_roce_udp_encap(const struct ib_device *device,3113						u32 port_num)3114{3115	return device->port_data[port_num].immutable.core_cap_flags &3116	       RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;3117}3118 3119static inline bool rdma_protocol_roce_eth_encap(const struct ib_device *device,3120						u32 port_num)3121{3122	return device->port_data[port_num].immutable.core_cap_flags &3123	       RDMA_CORE_CAP_PROT_ROCE;3124}3125 3126static inline bool rdma_protocol_iwarp(const struct ib_device *device,3127				       u32 port_num)3128{3129	return device->port_data[port_num].immutable.core_cap_flags &3130	       RDMA_CORE_CAP_PROT_IWARP;3131}3132 3133static inline bool rdma_ib_or_roce(const struct ib_device *device,3134				   u32 port_num)3135{3136	return rdma_protocol_ib(device, port_num) ||3137		rdma_protocol_roce(device, port_num);3138}3139 3140static inline bool rdma_protocol_raw_packet(const struct ib_device *device,3141					    u32 port_num)3142{3143	return device->port_data[port_num].immutable.core_cap_flags &3144	       RDMA_CORE_CAP_PROT_RAW_PACKET;3145}3146 3147static inline bool rdma_protocol_usnic(const struct ib_device *device,3148				       u32 port_num)3149{3150	return device->port_data[port_num].immutable.core_cap_flags &3151	       RDMA_CORE_CAP_PROT_USNIC;3152}3153 3154/**3155 * rdma_cap_ib_mad - Check if the port of a device supports Infiniband3156 * Management Datagrams.3157 * @device: Device to check3158 * @port_num: Port number to check3159 *3160 * Management Datagrams (MAD) are a required part of the InfiniBand3161 * specification and are supported on all InfiniBand devices.  A slightly3162 * extended version are also supported on OPA interfaces.3163 *3164 * Return: true if the port supports sending/receiving of MAD packets.3165 */3166static inline bool rdma_cap_ib_mad(const struct ib_device *device, u32 port_num)3167{3168	return device->port_data[port_num].immutable.core_cap_flags &3169	       RDMA_CORE_CAP_IB_MAD;3170}3171 3172/**3173 * rdma_cap_opa_mad - Check if the port of device provides support for OPA3174 * Management Datagrams.3175 * @device: Device to check3176 * @port_num: Port number to check3177 *3178 * Intel OmniPath devices extend and/or replace the InfiniBand Management3179 * datagrams with their own versions.  These OPA MADs share many but not all of3180 * the characteristics of InfiniBand MADs.3181 *3182 * OPA MADs differ in the following ways:3183 *3184 *    1) MADs are variable size up to 2K3185 *       IBTA defined MADs remain fixed at 256 bytes3186 *    2) OPA SMPs must carry valid PKeys3187 *    3) OPA SMP packets are a different format3188 *3189 * Return: true if the port supports OPA MAD packet formats.3190 */3191static inline bool rdma_cap_opa_mad(struct ib_device *device, u32 port_num)3192{3193	return device->port_data[port_num].immutable.core_cap_flags &3194		RDMA_CORE_CAP_OPA_MAD;3195}3196 3197/**3198 * rdma_cap_ib_smi - Check if the port of a device provides an Infiniband3199 * Subnet Management Agent (SMA) on the Subnet Management Interface (SMI).3200 * @device: Device to check3201 * @port_num: Port number to check3202 *3203 * Each InfiniBand node is required to provide a Subnet Management Agent3204 * that the subnet manager can access.  Prior to the fabric being fully3205 * configured by the subnet manager, the SMA is accessed via a well known3206 * interface called the Subnet Management Interface (SMI).  This interface3207 * uses directed route packets to communicate with the SM to get around the3208 * chicken and egg problem of the SM needing to know what's on the fabric3209 * in order to configure the fabric, and needing to configure the fabric in3210 * order to send packets to the devices on the fabric.  These directed3211 * route packets do not need the fabric fully configured in order to reach3212 * their destination.  The SMI is the only method allowed to send3213 * directed route packets on an InfiniBand fabric.3214 *3215 * Return: true if the port provides an SMI.3216 */3217static inline bool rdma_cap_ib_smi(const struct ib_device *device, u32 port_num)3218{3219	return device->port_data[port_num].immutable.core_cap_flags &3220	       RDMA_CORE_CAP_IB_SMI;3221}3222 3223/**3224 * rdma_cap_ib_cm - Check if the port of device has the capability Infiniband3225 * Communication Manager.3226 * @device: Device to check3227 * @port_num: Port number to check3228 *3229 * The InfiniBand Communication Manager is one of many pre-defined General3230 * Service Agents (GSA) that are accessed via the General Service3231 * Interface (GSI).  It's role is to facilitate establishment of connections3232 * between nodes as well as other management related tasks for established3233 * connections.3234 *3235 * Return: true if the port supports an IB CM (this does not guarantee that3236 * a CM is actually running however).3237 */3238static inline bool rdma_cap_ib_cm(const struct ib_device *device, u32 port_num)3239{3240	return device->port_data[port_num].immutable.core_cap_flags &3241	       RDMA_CORE_CAP_IB_CM;3242}3243 3244/**3245 * rdma_cap_iw_cm - Check if the port of device has the capability IWARP3246 * Communication Manager.3247 * @device: Device to check3248 * @port_num: Port number to check3249 *3250 * Similar to above, but specific to iWARP connections which have a different3251 * managment protocol than InfiniBand.3252 *3253 * Return: true if the port supports an iWARP CM (this does not guarantee that3254 * a CM is actually running however).3255 */3256static inline bool rdma_cap_iw_cm(const struct ib_device *device, u32 port_num)3257{3258	return device->port_data[port_num].immutable.core_cap_flags &3259	       RDMA_CORE_CAP_IW_CM;3260}3261 3262/**3263 * rdma_cap_ib_sa - Check if the port of device has the capability Infiniband3264 * Subnet Administration.3265 * @device: Device to check3266 * @port_num: Port number to check3267 *3268 * An InfiniBand Subnet Administration (SA) service is a pre-defined General3269 * Service Agent (GSA) provided by the Subnet Manager (SM).  On InfiniBand3270 * fabrics, devices should resolve routes to other hosts by contacting the3271 * SA to query the proper route.3272 *3273 * Return: true if the port should act as a client to the fabric Subnet3274 * Administration interface.  This does not imply that the SA service is3275 * running locally.3276 */3277static inline bool rdma_cap_ib_sa(const struct ib_device *device, u32 port_num)3278{3279	return device->port_data[port_num].immutable.core_cap_flags &3280	       RDMA_CORE_CAP_IB_SA;3281}3282 3283/**3284 * rdma_cap_ib_mcast - Check if the port of device has the capability Infiniband3285 * Multicast.3286 * @device: Device to check3287 * @port_num: Port number to check3288 *3289 * InfiniBand multicast registration is more complex than normal IPv4 or3290 * IPv6 multicast registration.  Each Host Channel Adapter must register3291 * with the Subnet Manager when it wishes to join a multicast group.  It3292 * should do so only once regardless of how many queue pairs it subscribes3293 * to this group.  And it should leave the group only after all queue pairs3294 * attached to the group have been detached.3295 *3296 * Return: true if the port must undertake the additional adminstrative3297 * overhead of registering/unregistering with the SM and tracking of the3298 * total number of queue pairs attached to the multicast group.3299 */3300static inline bool rdma_cap_ib_mcast(const struct ib_device *device,3301				     u32 port_num)3302{3303	return rdma_cap_ib_sa(device, port_num);3304}3305 3306/**3307 * rdma_cap_af_ib - Check if the port of device has the capability3308 * Native Infiniband Address.3309 * @device: Device to check3310 * @port_num: Port number to check3311 *3312 * InfiniBand addressing uses a port's GUID + Subnet Prefix to make a default3313 * GID.  RoCE uses a different mechanism, but still generates a GID via3314 * a prescribed mechanism and port specific data.3315 *3316 * Return: true if the port uses a GID address to identify devices on the3317 * network.3318 */3319static inline bool rdma_cap_af_ib(const struct ib_device *device, u32 port_num)3320{3321	return device->port_data[port_num].immutable.core_cap_flags &3322	       RDMA_CORE_CAP_AF_IB;3323}3324 3325/**3326 * rdma_cap_eth_ah - Check if the port of device has the capability3327 * Ethernet Address Handle.3328 * @device: Device to check3329 * @port_num: Port number to check3330 *3331 * RoCE is InfiniBand over Ethernet, and it uses a well defined technique3332 * to fabricate GIDs over Ethernet/IP specific addresses native to the3333 * port.  Normally, packet headers are generated by the sending host3334 * adapter, but when sending connectionless datagrams, we must manually3335 * inject the proper headers for the fabric we are communicating over.3336 *3337 * Return: true if we are running as a RoCE port and must force the3338 * addition of a Global Route Header built from our Ethernet Address3339 * Handle into our header list for connectionless packets.3340 */3341static inline bool rdma_cap_eth_ah(const struct ib_device *device, u32 port_num)3342{3343	return device->port_data[port_num].immutable.core_cap_flags &3344	       RDMA_CORE_CAP_ETH_AH;3345}3346 3347/**3348 * rdma_cap_opa_ah - Check if the port of device supports3349 * OPA Address handles3350 * @device: Device to check3351 * @port_num: Port number to check3352 *3353 * Return: true if we are running on an OPA device which supports3354 * the extended OPA addressing.3355 */3356static inline bool rdma_cap_opa_ah(struct ib_device *device, u32 port_num)3357{3358	return (device->port_data[port_num].immutable.core_cap_flags &3359		RDMA_CORE_CAP_OPA_AH) == RDMA_CORE_CAP_OPA_AH;3360}3361 3362/**3363 * rdma_max_mad_size - Return the max MAD size required by this RDMA Port.3364 *3365 * @device: Device3366 * @port_num: Port number3367 *3368 * This MAD size includes the MAD headers and MAD payload.  No other headers3369 * are included.3370 *3371 * Return the max MAD size required by the Port.  Will return 0 if the port3372 * does not support MADs3373 */3374static inline size_t rdma_max_mad_size(const struct ib_device *device,3375				       u32 port_num)3376{3377	return device->port_data[port_num].immutable.max_mad_size;3378}3379 3380/**3381 * rdma_cap_roce_gid_table - Check if the port of device uses roce_gid_table3382 * @device: Device to check3383 * @port_num: Port number to check3384 *3385 * RoCE GID table mechanism manages the various GIDs for a device.3386 *3387 * NOTE: if allocating the port's GID table has failed, this call will still3388 * return true, but any RoCE GID table API will fail.3389 *3390 * Return: true if the port uses RoCE GID table mechanism in order to manage3391 * its GIDs.3392 */3393static inline bool rdma_cap_roce_gid_table(const struct ib_device *device,3394					   u32 port_num)3395{3396	return rdma_protocol_roce(device, port_num) &&3397		device->ops.add_gid && device->ops.del_gid;3398}3399 3400/*3401 * Check if the device supports READ W/ INVALIDATE.3402 */3403static inline bool rdma_cap_read_inv(struct ib_device *dev, u32 port_num)3404{3405	/*3406	 * iWarp drivers must support READ W/ INVALIDATE.  No other protocol3407	 * has support for it yet.3408	 */3409	return rdma_protocol_iwarp(dev, port_num);3410}3411 3412/**3413 * rdma_core_cap_opa_port - Return whether the RDMA Port is OPA or not.3414 * @device: Device3415 * @port_num: 1 based Port number3416 *3417 * Return true if port is an Intel OPA port , false if not3418 */3419static inline bool rdma_core_cap_opa_port(struct ib_device *device,3420					  u32 port_num)3421{3422	return (device->port_data[port_num].immutable.core_cap_flags &3423		RDMA_CORE_PORT_INTEL_OPA) == RDMA_CORE_PORT_INTEL_OPA;3424}3425 3426/**3427 * rdma_mtu_enum_to_int - Return the mtu of the port as an integer value.3428 * @device: Device3429 * @port_num: Port number3430 * @mtu: enum value of MTU3431 *3432 * Return the MTU size supported by the port as an integer value. Will return3433 * -1 if enum value of mtu is not supported.3434 */3435static inline int rdma_mtu_enum_to_int(struct ib_device *device, u32 port,3436				       int mtu)3437{3438	if (rdma_core_cap_opa_port(device, port))3439		return opa_mtu_enum_to_int((enum opa_mtu)mtu);3440	else3441		return ib_mtu_enum_to_int((enum ib_mtu)mtu);3442}3443 3444/**3445 * rdma_mtu_from_attr - Return the mtu of the port from the port attribute.3446 * @device: Device3447 * @port_num: Port number3448 * @attr: port attribute3449 *3450 * Return the MTU size supported by the port as an integer value.3451 */3452static inline int rdma_mtu_from_attr(struct ib_device *device, u32 port,3453				     struct ib_port_attr *attr)3454{3455	if (rdma_core_cap_opa_port(device, port))3456		return attr->phys_mtu;3457	else3458		return ib_mtu_enum_to_int(attr->max_mtu);3459}3460 3461int ib_set_vf_link_state(struct ib_device *device, int vf, u32 port,3462			 int state);3463int ib_get_vf_config(struct ib_device *device, int vf, u32 port,3464		     struct ifla_vf_info *info);3465int ib_get_vf_stats(struct ib_device *device, int vf, u32 port,3466		    struct ifla_vf_stats *stats);3467int ib_get_vf_guid(struct ib_device *device, int vf, u32 port,3468		    struct ifla_vf_guid *node_guid,3469		    struct ifla_vf_guid *port_guid);3470int ib_set_vf_guid(struct ib_device *device, int vf, u32 port, u64 guid,3471		   int type);3472 3473int ib_query_pkey(struct ib_device *device,3474		  u32 port_num, u16 index, u16 *pkey);3475 3476int ib_modify_device(struct ib_device *device,3477		     int device_modify_mask,3478		     struct ib_device_modify *device_modify);3479 3480int ib_modify_port(struct ib_device *device,3481		   u32 port_num, int port_modify_mask,3482		   struct ib_port_modify *port_modify);3483 3484int ib_find_gid(struct ib_device *device, union ib_gid *gid,3485		u32 *port_num, u16 *index);3486 3487int ib_find_pkey(struct ib_device *device,3488		 u32 port_num, u16 pkey, u16 *index);3489 3490enum ib_pd_flags {3491	/*3492	 * Create a memory registration for all memory in the system and place3493	 * the rkey for it into pd->unsafe_global_rkey.  This can be used by3494	 * ULPs to avoid the overhead of dynamic MRs.3495	 *3496	 * This flag is generally considered unsafe and must only be used in3497	 * extremly trusted environments.  Every use of it will log a warning3498	 * in the kernel log.3499	 */3500	IB_PD_UNSAFE_GLOBAL_RKEY	= 0x01,3501};3502 3503struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,3504		const char *caller);3505 3506/**3507 * ib_alloc_pd - Allocates an unused protection domain.3508 * @device: The device on which to allocate the protection domain.3509 * @flags: protection domain flags3510 *3511 * A protection domain object provides an association between QPs, shared3512 * receive queues, address handles, memory regions, and memory windows.3513 *3514 * Every PD has a local_dma_lkey which can be used as the lkey value for local3515 * memory operations.3516 */3517#define ib_alloc_pd(device, flags) \3518	__ib_alloc_pd((device), (flags), KBUILD_MODNAME)3519 3520int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata);3521 3522/**3523 * ib_dealloc_pd - Deallocate kernel PD3524 * @pd: The protection domain3525 *3526 * NOTE: for user PD use ib_dealloc_pd_user with valid udata!3527 */3528static inline void ib_dealloc_pd(struct ib_pd *pd)3529{3530	int ret = ib_dealloc_pd_user(pd, NULL);3531 3532	WARN_ONCE(ret, "Destroy of kernel PD shouldn't fail");3533}3534 3535enum rdma_create_ah_flags {3536	/* In a sleepable context */3537	RDMA_CREATE_AH_SLEEPABLE = BIT(0),3538};3539 3540/**3541 * rdma_create_ah - Creates an address handle for the given address vector.3542 * @pd: The protection domain associated with the address handle.3543 * @ah_attr: The attributes of the address vector.3544 * @flags: Create address handle flags (see enum rdma_create_ah_flags).3545 *3546 * The address handle is used to reference a local or global destination3547 * in all UD QP post sends.3548 */3549struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,3550			     u32 flags);3551 3552/**3553 * rdma_create_user_ah - Creates an address handle for the given address vector.3554 * It resolves destination mac address for ah attribute of RoCE type.3555 * @pd: The protection domain associated with the address handle.3556 * @ah_attr: The attributes of the address vector.3557 * @udata: pointer to user's input output buffer information need by3558 *         provider driver.3559 *3560 * It returns 0 on success and returns appropriate error code on error.3561 * The address handle is used to reference a local or global destination3562 * in all UD QP post sends.3563 */3564struct ib_ah *rdma_create_user_ah(struct ib_pd *pd,3565				  struct rdma_ah_attr *ah_attr,3566				  struct ib_udata *udata);3567/**3568 * ib_get_gids_from_rdma_hdr - Get sgid and dgid from GRH or IPv4 header3569 *   work completion.3570 * @hdr: the L3 header to parse3571 * @net_type: type of header to parse3572 * @sgid: place to store source gid3573 * @dgid: place to store destination gid3574 */3575int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,3576			      enum rdma_network_type net_type,3577			      union ib_gid *sgid, union ib_gid *dgid);3578 3579/**3580 * ib_get_rdma_header_version - Get the header version3581 * @hdr: the L3 header to parse3582 */3583int ib_get_rdma_header_version(const union rdma_network_hdr *hdr);3584 3585/**3586 * ib_init_ah_attr_from_wc - Initializes address handle attributes from a3587 *   work completion.3588 * @device: Device on which the received message arrived.3589 * @port_num: Port on which the received message arrived.3590 * @wc: Work completion associated with the received message.3591 * @grh: References the received global route header.  This parameter is3592 *   ignored unless the work completion indicates that the GRH is valid.3593 * @ah_attr: Returned attributes that can be used when creating an address3594 *   handle for replying to the message.3595 * When ib_init_ah_attr_from_wc() returns success,3596 * (a) for IB link layer it optionally contains a reference to SGID attribute3597 * when GRH is present for IB link layer.3598 * (b) for RoCE link layer it contains a reference to SGID attribute.3599 * User must invoke rdma_cleanup_ah_attr_gid_attr() to release reference to SGID3600 * attributes which are initialized using ib_init_ah_attr_from_wc().3601 *3602 */3603int ib_init_ah_attr_from_wc(struct ib_device *device, u32 port_num,3604			    const struct ib_wc *wc, const struct ib_grh *grh,3605			    struct rdma_ah_attr *ah_attr);3606 3607/**3608 * ib_create_ah_from_wc - Creates an address handle associated with the3609 *   sender of the specified work completion.3610 * @pd: The protection domain associated with the address handle.3611 * @wc: Work completion information associated with a received message.3612 * @grh: References the received global route header.  This parameter is3613 *   ignored unless the work completion indicates that the GRH is valid.3614 * @port_num: The outbound port number to associate with the address.3615 *3616 * The address handle is used to reference a local or global destination3617 * in all UD QP post sends.3618 */3619struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,3620				   const struct ib_grh *grh, u32 port_num);3621 3622/**3623 * rdma_modify_ah - Modifies the address vector associated with an address3624 *   handle.3625 * @ah: The address handle to modify.3626 * @ah_attr: The new address vector attributes to associate with the3627 *   address handle.3628 */3629int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);3630 3631/**3632 * rdma_query_ah - Queries the address vector associated with an address3633 *   handle.3634 * @ah: The address handle to query.3635 * @ah_attr: The address vector attributes associated with the address3636 *   handle.3637 */3638int rdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);3639 3640enum rdma_destroy_ah_flags {3641	/* In a sleepable context */3642	RDMA_DESTROY_AH_SLEEPABLE = BIT(0),3643};3644 3645/**3646 * rdma_destroy_ah_user - Destroys an address handle.3647 * @ah: The address handle to destroy.3648 * @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).3649 * @udata: Valid user data or NULL for kernel objects3650 */3651int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata);3652 3653/**3654 * rdma_destroy_ah - Destroys an kernel address handle.3655 * @ah: The address handle to destroy.3656 * @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).3657 *3658 * NOTE: for user ah use rdma_destroy_ah_user with valid udata!3659 */3660static inline void rdma_destroy_ah(struct ib_ah *ah, u32 flags)3661{3662	int ret = rdma_destroy_ah_user(ah, flags, NULL);3663 3664	WARN_ONCE(ret, "Destroy of kernel AH shouldn't fail");3665}3666 3667struct ib_srq *ib_create_srq_user(struct ib_pd *pd,3668				  struct ib_srq_init_attr *srq_init_attr,3669				  struct ib_usrq_object *uobject,3670				  struct ib_udata *udata);3671static inline struct ib_srq *3672ib_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *srq_init_attr)3673{3674	if (!pd->device->ops.create_srq)3675		return ERR_PTR(-EOPNOTSUPP);3676 3677	return ib_create_srq_user(pd, srq_init_attr, NULL, NULL);3678}3679 3680/**3681 * ib_modify_srq - Modifies the attributes for the specified SRQ.3682 * @srq: The SRQ to modify.3683 * @srq_attr: On input, specifies the SRQ attributes to modify.  On output,3684 *   the current values of selected SRQ attributes are returned.3685 * @srq_attr_mask: A bit-mask used to specify which attributes of the SRQ3686 *   are being modified.3687 *3688 * The mask may contain IB_SRQ_MAX_WR to resize the SRQ and/or3689 * IB_SRQ_LIMIT to set the SRQ's limit and request notification when3690 * the number of receives queued drops below the limit.3691 */3692int ib_modify_srq(struct ib_srq *srq,3693		  struct ib_srq_attr *srq_attr,3694		  enum ib_srq_attr_mask srq_attr_mask);3695 3696/**3697 * ib_query_srq - Returns the attribute list and current values for the3698 *   specified SRQ.3699 * @srq: The SRQ to query.3700 * @srq_attr: The attributes of the specified SRQ.3701 */3702int ib_query_srq(struct ib_srq *srq,3703		 struct ib_srq_attr *srq_attr);3704 3705/**3706 * ib_destroy_srq_user - Destroys the specified SRQ.3707 * @srq: The SRQ to destroy.3708 * @udata: Valid user data or NULL for kernel objects3709 */3710int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata);3711 3712/**3713 * ib_destroy_srq - Destroys the specified kernel SRQ.3714 * @srq: The SRQ to destroy.3715 *3716 * NOTE: for user srq use ib_destroy_srq_user with valid udata!3717 */3718static inline void ib_destroy_srq(struct ib_srq *srq)3719{3720	int ret = ib_destroy_srq_user(srq, NULL);3721 3722	WARN_ONCE(ret, "Destroy of kernel SRQ shouldn't fail");3723}3724 3725/**3726 * ib_post_srq_recv - Posts a list of work requests to the specified SRQ.3727 * @srq: The SRQ to post the work request on.3728 * @recv_wr: A list of work requests to post on the receive queue.3729 * @bad_recv_wr: On an immediate failure, this parameter will reference3730 *   the work request that failed to be posted on the QP.3731 */3732static inline int ib_post_srq_recv(struct ib_srq *srq,3733				   const struct ib_recv_wr *recv_wr,3734				   const struct ib_recv_wr **bad_recv_wr)3735{3736	const struct ib_recv_wr *dummy;3737 3738	return srq->device->ops.post_srq_recv(srq, recv_wr,3739					      bad_recv_wr ? : &dummy);3740}3741 3742struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd,3743				  struct ib_qp_init_attr *qp_init_attr,3744				  const char *caller);3745/**3746 * ib_create_qp - Creates a kernel QP associated with the specific protection3747 * domain.3748 * @pd: The protection domain associated with the QP.3749 * @init_attr: A list of initial attributes required to create the3750 *   QP.  If QP creation succeeds, then the attributes are updated to3751 *   the actual capabilities of the created QP.3752 */3753static inline struct ib_qp *ib_create_qp(struct ib_pd *pd,3754					 struct ib_qp_init_attr *init_attr)3755{3756	return ib_create_qp_kernel(pd, init_attr, KBUILD_MODNAME);3757}3758 3759/**3760 * ib_modify_qp_with_udata - Modifies the attributes for the specified QP.3761 * @qp: The QP to modify.3762 * @attr: On input, specifies the QP attributes to modify.  On output,3763 *   the current values of selected QP attributes are returned.3764 * @attr_mask: A bit-mask used to specify which attributes of the QP3765 *   are being modified.3766 * @udata: pointer to user's input output buffer information3767 *   are being modified.3768 * It returns 0 on success and returns appropriate error code on error.3769 */3770int ib_modify_qp_with_udata(struct ib_qp *qp,3771			    struct ib_qp_attr *attr,3772			    int attr_mask,3773			    struct ib_udata *udata);3774 3775/**3776 * ib_modify_qp - Modifies the attributes for the specified QP and then3777 *   transitions the QP to the given state.3778 * @qp: The QP to modify.3779 * @qp_attr: On input, specifies the QP attributes to modify.  On output,3780 *   the current values of selected QP attributes are returned.3781 * @qp_attr_mask: A bit-mask used to specify which attributes of the QP3782 *   are being modified.3783 */3784int ib_modify_qp(struct ib_qp *qp,3785		 struct ib_qp_attr *qp_attr,3786		 int qp_attr_mask);3787 3788/**3789 * ib_query_qp - Returns the attribute list and current values for the3790 *   specified QP.3791 * @qp: The QP to query.3792 * @qp_attr: The attributes of the specified QP.3793 * @qp_attr_mask: A bit-mask used to select specific attributes to query.3794 * @qp_init_attr: Additional attributes of the selected QP.3795 *3796 * The qp_attr_mask may be used to limit the query to gathering only the3797 * selected attributes.3798 */3799int ib_query_qp(struct ib_qp *qp,3800		struct ib_qp_attr *qp_attr,3801		int qp_attr_mask,3802		struct ib_qp_init_attr *qp_init_attr);3803 3804/**3805 * ib_destroy_qp - Destroys the specified QP.3806 * @qp: The QP to destroy.3807 * @udata: Valid udata or NULL for kernel objects3808 */3809int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata);3810 3811/**3812 * ib_destroy_qp - Destroys the specified kernel QP.3813 * @qp: The QP to destroy.3814 *3815 * NOTE: for user qp use ib_destroy_qp_user with valid udata!3816 */3817static inline int ib_destroy_qp(struct ib_qp *qp)3818{3819	return ib_destroy_qp_user(qp, NULL);3820}3821 3822/**3823 * ib_open_qp - Obtain a reference to an existing sharable QP.3824 * @xrcd - XRC domain3825 * @qp_open_attr: Attributes identifying the QP to open.3826 *3827 * Returns a reference to a sharable QP.3828 */3829struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,3830			 struct ib_qp_open_attr *qp_open_attr);3831 3832/**3833 * ib_close_qp - Release an external reference to a QP.3834 * @qp: The QP handle to release3835 *3836 * The opened QP handle is released by the caller.  The underlying3837 * shared QP is not destroyed until all internal references are released.3838 */3839int ib_close_qp(struct ib_qp *qp);3840 3841/**3842 * ib_post_send - Posts a list of work requests to the send queue of3843 *   the specified QP.3844 * @qp: The QP to post the work request on.3845 * @send_wr: A list of work requests to post on the send queue.3846 * @bad_send_wr: On an immediate failure, this parameter will reference3847 *   the work request that failed to be posted on the QP.3848 *3849 * While IBA Vol. 1 section 11.4.1.1 specifies that if an immediate3850 * error is returned, the QP state shall not be affected,3851 * ib_post_send() will return an immediate error after queueing any3852 * earlier work requests in the list.3853 */3854static inline int ib_post_send(struct ib_qp *qp,3855			       const struct ib_send_wr *send_wr,3856			       const struct ib_send_wr **bad_send_wr)3857{3858	const struct ib_send_wr *dummy;3859 3860	return qp->device->ops.post_send(qp, send_wr, bad_send_wr ? : &dummy);3861}3862 3863/**3864 * ib_post_recv - Posts a list of work requests to the receive queue of3865 *   the specified QP.3866 * @qp: The QP to post the work request on.3867 * @recv_wr: A list of work requests to post on the receive queue.3868 * @bad_recv_wr: On an immediate failure, this parameter will reference3869 *   the work request that failed to be posted on the QP.3870 */3871static inline int ib_post_recv(struct ib_qp *qp,3872			       const struct ib_recv_wr *recv_wr,3873			       const struct ib_recv_wr **bad_recv_wr)3874{3875	const struct ib_recv_wr *dummy;3876 3877	return qp->device->ops.post_recv(qp, recv_wr, bad_recv_wr ? : &dummy);3878}3879 3880struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe,3881			    int comp_vector, enum ib_poll_context poll_ctx,3882			    const char *caller);3883static inline struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private,3884					int nr_cqe, int comp_vector,3885					enum ib_poll_context poll_ctx)3886{3887	return __ib_alloc_cq(dev, private, nr_cqe, comp_vector, poll_ctx,3888			     KBUILD_MODNAME);3889}3890 3891struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, void *private,3892				int nr_cqe, enum ib_poll_context poll_ctx,3893				const char *caller);3894 3895/**3896 * ib_alloc_cq_any: Allocate kernel CQ3897 * @dev: The IB device3898 * @private: Private data attached to the CQE3899 * @nr_cqe: Number of CQEs in the CQ3900 * @poll_ctx: Context used for polling the CQ3901 */3902static inline struct ib_cq *ib_alloc_cq_any(struct ib_device *dev,3903					    void *private, int nr_cqe,3904					    enum ib_poll_context poll_ctx)3905{3906	return __ib_alloc_cq_any(dev, private, nr_cqe, poll_ctx,3907				 KBUILD_MODNAME);3908}3909 3910void ib_free_cq(struct ib_cq *cq);3911int ib_process_cq_direct(struct ib_cq *cq, int budget);3912 3913/**3914 * ib_create_cq - Creates a CQ on the specified device.3915 * @device: The device on which to create the CQ.3916 * @comp_handler: A user-specified callback that is invoked when a3917 *   completion event occurs on the CQ.3918 * @event_handler: A user-specified callback that is invoked when an3919 *   asynchronous event not associated with a completion occurs on the CQ.3920 * @cq_context: Context associated with the CQ returned to the user via3921 *   the associated completion and event handlers.3922 * @cq_attr: The attributes the CQ should be created upon.3923 *3924 * Users can examine the cq structure to determine the actual CQ size.3925 */3926struct ib_cq *__ib_create_cq(struct ib_device *device,3927			     ib_comp_handler comp_handler,3928			     void (*event_handler)(struct ib_event *, void *),3929			     void *cq_context,3930			     const struct ib_cq_init_attr *cq_attr,3931			     const char *caller);3932#define ib_create_cq(device, cmp_hndlr, evt_hndlr, cq_ctxt, cq_attr) \3933	__ib_create_cq((device), (cmp_hndlr), (evt_hndlr), (cq_ctxt), (cq_attr), KBUILD_MODNAME)3934 3935/**3936 * ib_resize_cq - Modifies the capacity of the CQ.3937 * @cq: The CQ to resize.3938 * @cqe: The minimum size of the CQ.3939 *3940 * Users can examine the cq structure to determine the actual CQ size.3941 */3942int ib_resize_cq(struct ib_cq *cq, int cqe);3943 3944/**3945 * rdma_set_cq_moderation - Modifies moderation params of the CQ3946 * @cq: The CQ to modify.3947 * @cq_count: number of CQEs that will trigger an event3948 * @cq_period: max period of time in usec before triggering an event3949 *3950 */3951int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period);3952 3953/**3954 * ib_destroy_cq_user - Destroys the specified CQ.3955 * @cq: The CQ to destroy.3956 * @udata: Valid user data or NULL for kernel objects3957 */3958int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata);3959 3960/**3961 * ib_destroy_cq - Destroys the specified kernel CQ.3962 * @cq: The CQ to destroy.3963 *3964 * NOTE: for user cq use ib_destroy_cq_user with valid udata!3965 */3966static inline void ib_destroy_cq(struct ib_cq *cq)3967{3968	int ret = ib_destroy_cq_user(cq, NULL);3969 3970	WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail");3971}3972 3973/**3974 * ib_poll_cq - poll a CQ for completion(s)3975 * @cq:the CQ being polled3976 * @num_entries:maximum number of completions to return3977 * @wc:array of at least @num_entries &struct ib_wc where completions3978 *   will be returned3979 *3980 * Poll a CQ for (possibly multiple) completions.  If the return value3981 * is < 0, an error occurred.  If the return value is >= 0, it is the3982 * number of completions returned.  If the return value is3983 * non-negative and < num_entries, then the CQ was emptied.3984 */3985static inline int ib_poll_cq(struct ib_cq *cq, int num_entries,3986			     struct ib_wc *wc)3987{3988	return cq->device->ops.poll_cq(cq, num_entries, wc);3989}3990 3991/**3992 * ib_req_notify_cq - Request completion notification on a CQ.3993 * @cq: The CQ to generate an event for.3994 * @flags:3995 *   Must contain exactly one of %IB_CQ_SOLICITED or %IB_CQ_NEXT_COMP3996 *   to request an event on the next solicited event or next work3997 *   completion at any type, respectively. %IB_CQ_REPORT_MISSED_EVENTS3998 *   may also be |ed in to request a hint about missed events, as3999 *   described below.4000 *4001 * Return Value:4002 *    < 0 means an error occurred while requesting notification4003 *   == 0 means notification was requested successfully, and if4004 *        IB_CQ_REPORT_MISSED_EVENTS was passed in, then no events4005 *        were missed and it is safe to wait for another event.  In4006 *        this case is it guaranteed that any work completions added4007 *        to the CQ since the last CQ poll will trigger a completion4008 *        notification event.4009 *    > 0 is only returned if IB_CQ_REPORT_MISSED_EVENTS was passed4010 *        in.  It means that the consumer must poll the CQ again to4011 *        make sure it is empty to avoid missing an event because of a4012 *        race between requesting notification and an entry being4013 *        added to the CQ.  This return value means it is possible4014 *        (but not guaranteed) that a work completion has been added4015 *        to the CQ since the last poll without triggering a4016 *        completion notification event.4017 */4018static inline int ib_req_notify_cq(struct ib_cq *cq,4019				   enum ib_cq_notify_flags flags)4020{4021	return cq->device->ops.req_notify_cq(cq, flags);4022}4023 4024struct ib_cq *ib_cq_pool_get(struct ib_device *dev, unsigned int nr_cqe,4025			     int comp_vector_hint,4026			     enum ib_poll_context poll_ctx);4027 4028void ib_cq_pool_put(struct ib_cq *cq, unsigned int nr_cqe);4029 4030/*4031 * Drivers that don't need a DMA mapping at the RDMA layer, set dma_device to4032 * NULL. This causes the ib_dma* helpers to just stash the kernel virtual4033 * address into the dma address.4034 */4035static inline bool ib_uses_virt_dma(struct ib_device *dev)4036{4037	return IS_ENABLED(CONFIG_INFINIBAND_VIRT_DMA) && !dev->dma_device;4038}4039 4040/*4041 * Check if a IB device's underlying DMA mapping supports P2PDMA transfers.4042 */4043static inline bool ib_dma_pci_p2p_dma_supported(struct ib_device *dev)4044{4045	if (ib_uses_virt_dma(dev))4046		return false;4047 4048	return dma_pci_p2pdma_supported(dev->dma_device);4049}4050 4051/**4052 * ib_virt_dma_to_ptr - Convert a dma_addr to a kernel pointer4053 * @dma_addr: The DMA address4054 *4055 * Used by ib_uses_virt_dma() devices to get back to the kernel pointer after4056 * going through the dma_addr marshalling.4057 */4058static inline void *ib_virt_dma_to_ptr(u64 dma_addr)4059{4060	/* virt_dma mode maps the kvs's directly into the dma addr */4061	return (void *)(uintptr_t)dma_addr;4062}4063 4064/**4065 * ib_virt_dma_to_page - Convert a dma_addr to a struct page4066 * @dma_addr: The DMA address4067 *4068 * Used by ib_uses_virt_dma() device to get back to the struct page after going4069 * through the dma_addr marshalling.4070 */4071static inline struct page *ib_virt_dma_to_page(u64 dma_addr)4072{4073	return virt_to_page(ib_virt_dma_to_ptr(dma_addr));4074}4075 4076/**4077 * ib_dma_mapping_error - check a DMA addr for error4078 * @dev: The device for which the dma_addr was created4079 * @dma_addr: The DMA address to check4080 */4081static inline int ib_dma_mapping_error(struct ib_device *dev, u64 dma_addr)4082{4083	if (ib_uses_virt_dma(dev))4084		return 0;4085	return dma_mapping_error(dev->dma_device, dma_addr);4086}4087 4088/**4089 * ib_dma_map_single - Map a kernel virtual address to DMA address4090 * @dev: The device for which the dma_addr is to be created4091 * @cpu_addr: The kernel virtual address4092 * @size: The size of the region in bytes4093 * @direction: The direction of the DMA4094 */4095static inline u64 ib_dma_map_single(struct ib_device *dev,4096				    void *cpu_addr, size_t size,4097				    enum dma_data_direction direction)4098{4099	if (ib_uses_virt_dma(dev))4100		return (uintptr_t)cpu_addr;4101	return dma_map_single(dev->dma_device, cpu_addr, size, direction);4102}4103 4104/**4105 * ib_dma_unmap_single - Destroy a mapping created by ib_dma_map_single()4106 * @dev: The device for which the DMA address was created4107 * @addr: The DMA address4108 * @size: The size of the region in bytes4109 * @direction: The direction of the DMA4110 */4111static inline void ib_dma_unmap_single(struct ib_device *dev,4112				       u64 addr, size_t size,4113				       enum dma_data_direction direction)4114{4115	if (!ib_uses_virt_dma(dev))4116		dma_unmap_single(dev->dma_device, addr, size, direction);4117}4118 4119/**4120 * ib_dma_map_page - Map a physical page to DMA address4121 * @dev: The device for which the dma_addr is to be created4122 * @page: The page to be mapped4123 * @offset: The offset within the page4124 * @size: The size of the region in bytes4125 * @direction: The direction of the DMA4126 */4127static inline u64 ib_dma_map_page(struct ib_device *dev,4128				  struct page *page,4129				  unsigned long offset,4130				  size_t size,4131					 enum dma_data_direction direction)4132{4133	if (ib_uses_virt_dma(dev))4134		return (uintptr_t)(page_address(page) + offset);4135	return dma_map_page(dev->dma_device, page, offset, size, direction);4136}4137 4138/**4139 * ib_dma_unmap_page - Destroy a mapping created by ib_dma_map_page()4140 * @dev: The device for which the DMA address was created4141 * @addr: The DMA address4142 * @size: The size of the region in bytes4143 * @direction: The direction of the DMA4144 */4145static inline void ib_dma_unmap_page(struct ib_device *dev,4146				     u64 addr, size_t size,4147				     enum dma_data_direction direction)4148{4149	if (!ib_uses_virt_dma(dev))4150		dma_unmap_page(dev->dma_device, addr, size, direction);4151}4152 4153int ib_dma_virt_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents);4154static inline int ib_dma_map_sg_attrs(struct ib_device *dev,4155				      struct scatterlist *sg, int nents,4156				      enum dma_data_direction direction,4157				      unsigned long dma_attrs)4158{4159	if (ib_uses_virt_dma(dev))4160		return ib_dma_virt_map_sg(dev, sg, nents);4161	return dma_map_sg_attrs(dev->dma_device, sg, nents, direction,4162				dma_attrs);4163}4164 4165static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev,4166					 struct scatterlist *sg, int nents,4167					 enum dma_data_direction direction,4168					 unsigned long dma_attrs)4169{4170	if (!ib_uses_virt_dma(dev))4171		dma_unmap_sg_attrs(dev->dma_device, sg, nents, direction,4172				   dma_attrs);4173}4174 4175/**4176 * ib_dma_map_sgtable_attrs - Map a scatter/gather table to DMA addresses4177 * @dev: The device for which the DMA addresses are to be created4178 * @sg: The sg_table object describing the buffer4179 * @direction: The direction of the DMA4180 * @attrs: Optional DMA attributes for the map operation4181 */4182static inline int ib_dma_map_sgtable_attrs(struct ib_device *dev,4183					   struct sg_table *sgt,4184					   enum dma_data_direction direction,4185					   unsigned long dma_attrs)4186{4187	int nents;4188 4189	if (ib_uses_virt_dma(dev)) {4190		nents = ib_dma_virt_map_sg(dev, sgt->sgl, sgt->orig_nents);4191		if (!nents)4192			return -EIO;4193		sgt->nents = nents;4194		return 0;4195	}4196	return dma_map_sgtable(dev->dma_device, sgt, direction, dma_attrs);4197}4198 4199static inline void ib_dma_unmap_sgtable_attrs(struct ib_device *dev,4200					      struct sg_table *sgt,4201					      enum dma_data_direction direction,4202					      unsigned long dma_attrs)4203{4204	if (!ib_uses_virt_dma(dev))4205		dma_unmap_sgtable(dev->dma_device, sgt, direction, dma_attrs);4206}4207 4208/**4209 * ib_dma_map_sg - Map a scatter/gather list to DMA addresses4210 * @dev: The device for which the DMA addresses are to be created4211 * @sg: The array of scatter/gather entries4212 * @nents: The number of scatter/gather entries4213 * @direction: The direction of the DMA4214 */4215static inline int ib_dma_map_sg(struct ib_device *dev,4216				struct scatterlist *sg, int nents,4217				enum dma_data_direction direction)4218{4219	return ib_dma_map_sg_attrs(dev, sg, nents, direction, 0);4220}4221 4222/**4223 * ib_dma_unmap_sg - Unmap a scatter/gather list of DMA addresses4224 * @dev: The device for which the DMA addresses were created4225 * @sg: The array of scatter/gather entries4226 * @nents: The number of scatter/gather entries4227 * @direction: The direction of the DMA4228 */4229static inline void ib_dma_unmap_sg(struct ib_device *dev,4230				   struct scatterlist *sg, int nents,4231				   enum dma_data_direction direction)4232{4233	ib_dma_unmap_sg_attrs(dev, sg, nents, direction, 0);4234}4235 4236/**4237 * ib_dma_max_seg_size - Return the size limit of a single DMA transfer4238 * @dev: The device to query4239 *4240 * The returned value represents a size in bytes.4241 */4242static inline unsigned int ib_dma_max_seg_size(struct ib_device *dev)4243{4244	if (ib_uses_virt_dma(dev))4245		return UINT_MAX;4246	return dma_get_max_seg_size(dev->dma_device);4247}4248 4249/**4250 * ib_dma_sync_single_for_cpu - Prepare DMA region to be accessed by CPU4251 * @dev: The device for which the DMA address was created4252 * @addr: The DMA address4253 * @size: The size of the region in bytes4254 * @dir: The direction of the DMA4255 */4256static inline void ib_dma_sync_single_for_cpu(struct ib_device *dev,4257					      u64 addr,4258					      size_t size,4259					      enum dma_data_direction dir)4260{4261	if (!ib_uses_virt_dma(dev))4262		dma_sync_single_for_cpu(dev->dma_device, addr, size, dir);4263}4264 4265/**4266 * ib_dma_sync_single_for_device - Prepare DMA region to be accessed by device4267 * @dev: The device for which the DMA address was created4268 * @addr: The DMA address4269 * @size: The size of the region in bytes4270 * @dir: The direction of the DMA4271 */4272static inline void ib_dma_sync_single_for_device(struct ib_device *dev,4273						 u64 addr,4274						 size_t size,4275						 enum dma_data_direction dir)4276{4277	if (!ib_uses_virt_dma(dev))4278		dma_sync_single_for_device(dev->dma_device, addr, size, dir);4279}4280 4281/* ib_reg_user_mr - register a memory region for virtual addresses from kernel4282 * space. This function should be called when 'current' is the owning MM.4283 */4284struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,4285			     u64 virt_addr, int mr_access_flags);4286 4287/* ib_advise_mr -  give an advice about an address range in a memory region */4288int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,4289		 u32 flags, struct ib_sge *sg_list, u32 num_sge);4290/**4291 * ib_dereg_mr_user - Deregisters a memory region and removes it from the4292 *   HCA translation table.4293 * @mr: The memory region to deregister.4294 * @udata: Valid user data or NULL for kernel object4295 *4296 * This function can fail, if the memory region has memory windows bound to it.4297 */4298int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata);4299 4300/**4301 * ib_dereg_mr - Deregisters a kernel memory region and removes it from the4302 *   HCA translation table.4303 * @mr: The memory region to deregister.4304 *4305 * This function can fail, if the memory region has memory windows bound to it.4306 *4307 * NOTE: for user mr use ib_dereg_mr_user with valid udata!4308 */4309static inline int ib_dereg_mr(struct ib_mr *mr)4310{4311	return ib_dereg_mr_user(mr, NULL);4312}4313 4314struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,4315			  u32 max_num_sg);4316 4317struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,4318				    u32 max_num_data_sg,4319				    u32 max_num_meta_sg);4320 4321/**4322 * ib_update_fast_reg_key - updates the key portion of the fast_reg MR4323 *   R_Key and L_Key.4324 * @mr - struct ib_mr pointer to be updated.4325 * @newkey - new key to be used.4326 */4327static inline void ib_update_fast_reg_key(struct ib_mr *mr, u8 newkey)4328{4329	mr->lkey = (mr->lkey & 0xffffff00) | newkey;4330	mr->rkey = (mr->rkey & 0xffffff00) | newkey;4331}4332 4333/**4334 * ib_inc_rkey - increments the key portion of the given rkey. Can be used4335 * for calculating a new rkey for type 2 memory windows.4336 * @rkey - the rkey to increment.4337 */4338static inline u32 ib_inc_rkey(u32 rkey)4339{4340	const u32 mask = 0x000000ff;4341	return ((rkey + 1) & mask) | (rkey & ~mask);4342}4343 4344/**4345 * ib_attach_mcast - Attaches the specified QP to a multicast group.4346 * @qp: QP to attach to the multicast group.  The QP must be type4347 *   IB_QPT_UD.4348 * @gid: Multicast group GID.4349 * @lid: Multicast group LID in host byte order.4350 *4351 * In order to send and receive multicast packets, subnet4352 * administration must have created the multicast group and configured4353 * the fabric appropriately.  The port associated with the specified4354 * QP must also be a member of the multicast group.4355 */4356int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);4357 4358/**4359 * ib_detach_mcast - Detaches the specified QP from a multicast group.4360 * @qp: QP to detach from the multicast group.4361 * @gid: Multicast group GID.4362 * @lid: Multicast group LID in host byte order.4363 */4364int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);4365 4366struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device,4367				   struct inode *inode, struct ib_udata *udata);4368int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata);4369 4370static inline int ib_check_mr_access(struct ib_device *ib_dev,4371				     unsigned int flags)4372{4373	u64 device_cap = ib_dev->attrs.device_cap_flags;4374 4375	/*4376	 * Local write permission is required if remote write or4377	 * remote atomic permission is also requested.4378	 */4379	if (flags & (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE) &&4380	    !(flags & IB_ACCESS_LOCAL_WRITE))4381		return -EINVAL;4382 4383	if (flags & ~IB_ACCESS_SUPPORTED)4384		return -EINVAL;4385 4386	if (flags & IB_ACCESS_ON_DEMAND &&4387	    !(ib_dev->attrs.kernel_cap_flags & IBK_ON_DEMAND_PAGING))4388		return -EOPNOTSUPP;4389 4390	if ((flags & IB_ACCESS_FLUSH_GLOBAL &&4391	    !(device_cap & IB_DEVICE_FLUSH_GLOBAL)) ||4392	    (flags & IB_ACCESS_FLUSH_PERSISTENT &&4393	    !(device_cap & IB_DEVICE_FLUSH_PERSISTENT)))4394		return -EOPNOTSUPP;4395 4396	return 0;4397}4398 4399static inline bool ib_access_writable(int access_flags)4400{4401	/*4402	 * We have writable memory backing the MR if any of the following4403	 * access flags are set.  "Local write" and "remote write" obviously4404	 * require write access.  "Remote atomic" can do things like fetch and4405	 * add, which will modify memory, and "MW bind" can change permissions4406	 * by binding a window.4407	 */4408	return access_flags &4409		(IB_ACCESS_LOCAL_WRITE   | IB_ACCESS_REMOTE_WRITE |4410		 IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_MW_BIND);4411}4412 4413/**4414 * ib_check_mr_status: lightweight check of MR status.4415 *     This routine may provide status checks on a selected4416 *     ib_mr. first use is for signature status check.4417 *4418 * @mr: A memory region.4419 * @check_mask: Bitmask of which checks to perform from4420 *     ib_mr_status_check enumeration.4421 * @mr_status: The container of relevant status checks.4422 *     failed checks will be indicated in the status bitmask4423 *     and the relevant info shall be in the error item.4424 */4425int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,4426		       struct ib_mr_status *mr_status);4427 4428/**4429 * ib_device_try_get: Hold a registration lock4430 * device: The device to lock4431 *4432 * A device under an active registration lock cannot become unregistered. It4433 * is only possible to obtain a registration lock on a device that is fully4434 * registered, otherwise this function returns false.4435 *4436 * The registration lock is only necessary for actions which require the4437 * device to still be registered. Uses that only require the device pointer to4438 * be valid should use get_device(&ibdev->dev) to hold the memory.4439 *4440 */4441static inline bool ib_device_try_get(struct ib_device *dev)4442{4443	return refcount_inc_not_zero(&dev->refcount);4444}4445 4446void ib_device_put(struct ib_device *device);4447struct ib_device *ib_device_get_by_netdev(struct net_device *ndev,4448					  enum rdma_driver_id driver_id);4449struct ib_device *ib_device_get_by_name(const char *name,4450					enum rdma_driver_id driver_id);4451struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, u32 port,4452					    u16 pkey, const union ib_gid *gid,4453					    const struct sockaddr *addr);4454int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev,4455			 unsigned int port);4456struct net_device *ib_device_get_netdev(struct ib_device *ib_dev,4457					u32 port);4458struct ib_wq *ib_create_wq(struct ib_pd *pd,4459			   struct ib_wq_init_attr *init_attr);4460int ib_destroy_wq_user(struct ib_wq *wq, struct ib_udata *udata);4461 4462int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,4463		 unsigned int *sg_offset, unsigned int page_size);4464int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,4465		    int data_sg_nents, unsigned int *data_sg_offset,4466		    struct scatterlist *meta_sg, int meta_sg_nents,4467		    unsigned int *meta_sg_offset, unsigned int page_size);4468 4469static inline int4470ib_map_mr_sg_zbva(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,4471		  unsigned int *sg_offset, unsigned int page_size)4472{4473	int n;4474 4475	n = ib_map_mr_sg(mr, sg, sg_nents, sg_offset, page_size);4476	mr->iova = 0;4477 4478	return n;4479}4480 4481int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,4482		unsigned int *sg_offset, int (*set_page)(struct ib_mr *, u64));4483 4484void ib_drain_rq(struct ib_qp *qp);4485void ib_drain_sq(struct ib_qp *qp);4486void ib_drain_qp(struct ib_qp *qp);4487 4488int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed,4489		     u8 *width);4490 4491static inline u8 *rdma_ah_retrieve_dmac(struct rdma_ah_attr *attr)4492{4493	if (attr->type == RDMA_AH_ATTR_TYPE_ROCE)4494		return attr->roce.dmac;4495	return NULL;4496}4497 4498static inline void rdma_ah_set_dlid(struct rdma_ah_attr *attr, u32 dlid)4499{4500	if (attr->type == RDMA_AH_ATTR_TYPE_IB)4501		attr->ib.dlid = (u16)dlid;4502	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)4503		attr->opa.dlid = dlid;4504}4505 4506static inline u32 rdma_ah_get_dlid(const struct rdma_ah_attr *attr)4507{4508	if (attr->type == RDMA_AH_ATTR_TYPE_IB)4509		return attr->ib.dlid;4510	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)4511		return attr->opa.dlid;4512	return 0;4513}4514 4515static inline void rdma_ah_set_sl(struct rdma_ah_attr *attr, u8 sl)4516{4517	attr->sl = sl;4518}4519 4520static inline u8 rdma_ah_get_sl(const struct rdma_ah_attr *attr)4521{4522	return attr->sl;4523}4524 4525static inline void rdma_ah_set_path_bits(struct rdma_ah_attr *attr,4526					 u8 src_path_bits)4527{4528	if (attr->type == RDMA_AH_ATTR_TYPE_IB)4529		attr->ib.src_path_bits = src_path_bits;4530	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)4531		attr->opa.src_path_bits = src_path_bits;4532}4533 4534static inline u8 rdma_ah_get_path_bits(const struct rdma_ah_attr *attr)4535{4536	if (attr->type == RDMA_AH_ATTR_TYPE_IB)4537		return attr->ib.src_path_bits;4538	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)4539		return attr->opa.src_path_bits;4540	return 0;4541}4542 4543static inline void rdma_ah_set_make_grd(struct rdma_ah_attr *attr,4544					bool make_grd)4545{4546	if (attr->type == RDMA_AH_ATTR_TYPE_OPA)4547		attr->opa.make_grd = make_grd;4548}4549 4550static inline bool rdma_ah_get_make_grd(const struct rdma_ah_attr *attr)4551{4552	if (attr->type == RDMA_AH_ATTR_TYPE_OPA)4553		return attr->opa.make_grd;4554	return false;4555}4556 4557static inline void rdma_ah_set_port_num(struct rdma_ah_attr *attr, u32 port_num)4558{4559	attr->port_num = port_num;4560}4561 4562static inline u32 rdma_ah_get_port_num(const struct rdma_ah_attr *attr)4563{4564	return attr->port_num;4565}4566 4567static inline void rdma_ah_set_static_rate(struct rdma_ah_attr *attr,4568					   u8 static_rate)4569{4570	attr->static_rate = static_rate;4571}4572 4573static inline u8 rdma_ah_get_static_rate(const struct rdma_ah_attr *attr)4574{4575	return attr->static_rate;4576}4577 4578static inline void rdma_ah_set_ah_flags(struct rdma_ah_attr *attr,4579					enum ib_ah_flags flag)4580{4581	attr->ah_flags = flag;4582}4583 4584static inline enum ib_ah_flags4585		rdma_ah_get_ah_flags(const struct rdma_ah_attr *attr)4586{4587	return attr->ah_flags;4588}4589 4590static inline const struct ib_global_route4591		*rdma_ah_read_grh(const struct rdma_ah_attr *attr)4592{4593	return &attr->grh;4594}4595 4596/*To retrieve and modify the grh */4597static inline struct ib_global_route4598		*rdma_ah_retrieve_grh(struct rdma_ah_attr *attr)4599{4600	return &attr->grh;4601}4602 4603static inline void rdma_ah_set_dgid_raw(struct rdma_ah_attr *attr, void *dgid)4604{4605	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);4606 4607	memcpy(grh->dgid.raw, dgid, sizeof(grh->dgid));4608}4609 4610static inline void rdma_ah_set_subnet_prefix(struct rdma_ah_attr *attr,4611					     __be64 prefix)4612{4613	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);4614 4615	grh->dgid.global.subnet_prefix = prefix;4616}4617 4618static inline void rdma_ah_set_interface_id(struct rdma_ah_attr *attr,4619					    __be64 if_id)4620{4621	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);4622 4623	grh->dgid.global.interface_id = if_id;4624}4625 4626static inline void rdma_ah_set_grh(struct rdma_ah_attr *attr,4627				   union ib_gid *dgid, u32 flow_label,4628				   u8 sgid_index, u8 hop_limit,4629				   u8 traffic_class)4630{4631	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);4632 4633	attr->ah_flags = IB_AH_GRH;4634	if (dgid)4635		grh->dgid = *dgid;4636	grh->flow_label = flow_label;4637	grh->sgid_index = sgid_index;4638	grh->hop_limit = hop_limit;4639	grh->traffic_class = traffic_class;4640	grh->sgid_attr = NULL;4641}4642 4643void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr);4644void rdma_move_grh_sgid_attr(struct rdma_ah_attr *attr, union ib_gid *dgid,4645			     u32 flow_label, u8 hop_limit, u8 traffic_class,4646			     const struct ib_gid_attr *sgid_attr);4647void rdma_copy_ah_attr(struct rdma_ah_attr *dest,4648		       const struct rdma_ah_attr *src);4649void rdma_replace_ah_attr(struct rdma_ah_attr *old,4650			  const struct rdma_ah_attr *new);4651void rdma_move_ah_attr(struct rdma_ah_attr *dest, struct rdma_ah_attr *src);4652 4653/**4654 * rdma_ah_find_type - Return address handle type.4655 *4656 * @dev: Device to be checked4657 * @port_num: Port number4658 */4659static inline enum rdma_ah_attr_type rdma_ah_find_type(struct ib_device *dev,4660						       u32 port_num)4661{4662	if (rdma_protocol_roce(dev, port_num))4663		return RDMA_AH_ATTR_TYPE_ROCE;4664	if (rdma_protocol_ib(dev, port_num)) {4665		if (rdma_cap_opa_ah(dev, port_num))4666			return RDMA_AH_ATTR_TYPE_OPA;4667		return RDMA_AH_ATTR_TYPE_IB;4668	}4669	if (dev->type == RDMA_DEVICE_TYPE_SMI)4670		return RDMA_AH_ATTR_TYPE_IB;4671 4672	return RDMA_AH_ATTR_TYPE_UNDEFINED;4673}4674 4675/**4676 * ib_lid_cpu16 - Return lid in 16bit CPU encoding.4677 *     In the current implementation the only way to4678 *     get the 32bit lid is from other sources for OPA.4679 *     For IB, lids will always be 16bits so cast the4680 *     value accordingly.4681 *4682 * @lid: A 32bit LID4683 */4684static inline u16 ib_lid_cpu16(u32 lid)4685{4686	WARN_ON_ONCE(lid & 0xFFFF0000);4687	return (u16)lid;4688}4689 4690/**4691 * ib_lid_be16 - Return lid in 16bit BE encoding.4692 *4693 * @lid: A 32bit LID4694 */4695static inline __be16 ib_lid_be16(u32 lid)4696{4697	WARN_ON_ONCE(lid & 0xFFFF0000);4698	return cpu_to_be16((u16)lid);4699}4700 4701/**4702 * ib_get_vector_affinity - Get the affinity mappings of a given completion4703 *   vector4704 * @device:         the rdma device4705 * @comp_vector:    index of completion vector4706 *4707 * Returns NULL on failure, otherwise a corresponding cpu map of the4708 * completion vector (returns all-cpus map if the device driver doesn't4709 * implement get_vector_affinity).4710 */4711static inline const struct cpumask *4712ib_get_vector_affinity(struct ib_device *device, int comp_vector)4713{4714	if (comp_vector < 0 || comp_vector >= device->num_comp_vectors ||4715	    !device->ops.get_vector_affinity)4716		return NULL;4717 4718	return device->ops.get_vector_affinity(device, comp_vector);4719 4720}4721 4722/**4723 * rdma_roce_rescan_device - Rescan all of the network devices in the system4724 * and add their gids, as needed, to the relevant RoCE devices.4725 *4726 * @device:         the rdma device4727 */4728void rdma_roce_rescan_device(struct ib_device *ibdev);4729 4730struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile);4731 4732int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs);4733 4734struct net_device *rdma_alloc_netdev(struct ib_device *device, u32 port_num,4735				     enum rdma_netdev_t type, const char *name,4736				     unsigned char name_assign_type,4737				     void (*setup)(struct net_device *));4738 4739int rdma_init_netdev(struct ib_device *device, u32 port_num,4740		     enum rdma_netdev_t type, const char *name,4741		     unsigned char name_assign_type,4742		     void (*setup)(struct net_device *),4743		     struct net_device *netdev);4744 4745/**4746 * rdma_device_to_ibdev - Get ib_device pointer from device pointer4747 *4748 * @device:	device pointer for which ib_device pointer to retrieve4749 *4750 * rdma_device_to_ibdev() retrieves ib_device pointer from device.4751 *4752 */4753static inline struct ib_device *rdma_device_to_ibdev(struct device *device)4754{4755	struct ib_core_device *coredev =4756		container_of(device, struct ib_core_device, dev);4757 4758	return coredev->owner;4759}4760 4761/**4762 * ibdev_to_node - return the NUMA node for a given ib_device4763 * @dev:	device to get the NUMA node for.4764 */4765static inline int ibdev_to_node(struct ib_device *ibdev)4766{4767	struct device *parent = ibdev->dev.parent;4768 4769	if (!parent)4770		return NUMA_NO_NODE;4771	return dev_to_node(parent);4772}4773 4774/**4775 * rdma_device_to_drv_device - Helper macro to reach back to driver's4776 *			       ib_device holder structure from device pointer.4777 *4778 * NOTE: New drivers should not make use of this API; This API is only for4779 * existing drivers who have exposed sysfs entries using4780 * ops->device_group.4781 */4782#define rdma_device_to_drv_device(dev, drv_dev_struct, ibdev_member)           \4783	container_of(rdma_device_to_ibdev(dev), drv_dev_struct, ibdev_member)4784 4785bool rdma_dev_access_netns(const struct ib_device *device,4786			   const struct net *net);4787 4788#define IB_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000)4789#define IB_ROCE_UDP_ENCAP_VALID_PORT_MAX (0xFFFF)4790#define IB_GRH_FLOWLABEL_MASK (0x000FFFFF)4791 4792/**4793 * rdma_flow_label_to_udp_sport - generate a RoCE v2 UDP src port value based4794 *                               on the flow_label4795 *4796 * This function will convert the 20 bit flow_label input to a valid RoCE v24797 * UDP src port 14 bit value. All RoCE V2 drivers should use this same4798 * convention.4799 */4800static inline u16 rdma_flow_label_to_udp_sport(u32 fl)4801{4802	u32 fl_low = fl & 0x03fff, fl_high = fl & 0xFC000;4803 4804	fl_low ^= fl_high >> 14;4805	return (u16)(fl_low | IB_ROCE_UDP_ENCAP_VALID_PORT_MIN);4806}4807 4808/**4809 * rdma_calc_flow_label - generate a RDMA symmetric flow label value based on4810 *                        local and remote qpn values4811 *4812 * This function folded the multiplication results of two qpns, 24 bit each,4813 * fields, and converts it to a 20 bit results.4814 *4815 * This function will create symmetric flow_label value based on the local4816 * and remote qpn values. this will allow both the requester and responder4817 * to calculate the same flow_label for a given connection.4818 *4819 * This helper function should be used by driver in case the upper layer4820 * provide a zero flow_label value. This is to improve entropy of RDMA4821 * traffic in the network.4822 */4823static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn)4824{4825	u64 v = (u64)lqpn * rqpn;4826 4827	v ^= v >> 20;4828	v ^= v >> 40;4829 4830	return (u32)(v & IB_GRH_FLOWLABEL_MASK);4831}4832 4833/**4834 * rdma_get_udp_sport - Calculate and set UDP source port based on the flow4835 *                      label. If flow label is not defined in GRH then4836 *                      calculate it based on lqpn/rqpn.4837 *4838 * @fl:                 flow label from GRH4839 * @lqpn:               local qp number4840 * @rqpn:               remote qp number4841 */4842static inline u16 rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)4843{4844	if (!fl)4845		fl = rdma_calc_flow_label(lqpn, rqpn);4846 4847	return rdma_flow_label_to_udp_sport(fl);4848}4849 4850const struct ib_port_immutable*4851ib_port_immutable_read(struct ib_device *dev, unsigned int port);4852 4853/** ib_add_sub_device - Add a sub IB device on an existing one4854 *4855 * @parent: The IB device that needs to add a sub device4856 * @type: The type of the new sub device4857 * @name: The name of the new sub device4858 *4859 *4860 * Return 0 on success, an error code otherwise4861 */4862int ib_add_sub_device(struct ib_device *parent,4863		      enum rdma_nl_dev_type type,4864		      const char *name);4865 4866 4867/** ib_del_sub_device_and_put - Delect an IB sub device while holding a 'get'4868 *4869 * @sub: The sub device that is going to be deleted4870 *4871 * Return 0 on success, an error code otherwise4872 */4873int ib_del_sub_device_and_put(struct ib_device *sub);4874 4875static inline void ib_mark_name_assigned_by_user(struct ib_device *ibdev)4876{4877	ibdev->name_assign_type = RDMA_NAME_ASSIGN_TYPE_USER;4878}4879 4880#endif /* IB_VERBS_H */4881