477 lines · c
1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */2/*3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.5 */6 7#ifndef RXE_VERBS_H8#define RXE_VERBS_H9 10#include <linux/interrupt.h>11#include <linux/workqueue.h>12#include "rxe_pool.h"13#include "rxe_task.h"14#include "rxe_hw_counters.h"15 16static inline int pkey_match(u16 key1, u16 key2)17{18 return (((key1 & 0x7fff) != 0) &&19 ((key1 & 0x7fff) == (key2 & 0x7fff)) &&20 ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0;21}22 23/* Return >0 if psn_a > psn_b24 * 0 if psn_a == psn_b25 * <0 if psn_a < psn_b26 */27static inline int psn_compare(u32 psn_a, u32 psn_b)28{29 s32 diff;30 31 diff = (psn_a - psn_b) << 8;32 return diff;33}34 35struct rxe_ucontext {36 struct ib_ucontext ibuc;37 struct rxe_pool_elem elem;38};39 40struct rxe_pd {41 struct ib_pd ibpd;42 struct rxe_pool_elem elem;43};44 45struct rxe_ah {46 struct ib_ah ibah;47 struct rxe_pool_elem elem;48 struct rxe_av av;49 bool is_user;50 int ah_num;51};52 53struct rxe_cqe {54 union {55 struct ib_wc ibwc;56 struct ib_uverbs_wc uibwc;57 };58};59 60struct rxe_cq {61 struct ib_cq ibcq;62 struct rxe_pool_elem elem;63 struct rxe_queue *queue;64 spinlock_t cq_lock;65 u8 notify;66 bool is_user;67 atomic_t num_wq;68};69 70enum wqe_state {71 wqe_state_posted,72 wqe_state_processing,73 wqe_state_pending,74 wqe_state_done,75 wqe_state_error,76};77 78struct rxe_sq {79 int max_wr;80 int max_sge;81 int max_inline;82 spinlock_t sq_lock; /* guard queue */83 struct rxe_queue *queue;84};85 86struct rxe_rq {87 int max_wr;88 int max_sge;89 spinlock_t producer_lock; /* guard queue producer */90 spinlock_t consumer_lock; /* guard queue consumer */91 struct rxe_queue *queue;92};93 94struct rxe_srq {95 struct ib_srq ibsrq;96 struct rxe_pool_elem elem;97 struct rxe_pd *pd;98 struct rxe_rq rq;99 u32 srq_num;100 101 int limit;102 int error;103};104 105struct rxe_req_info {106 int wqe_index;107 u32 psn;108 int opcode;109 atomic_t rd_atomic;110 int wait_fence;111 int need_rd_atomic;112 int wait_psn;113 int need_retry;114 int wait_for_rnr_timer;115 int noack_pkts;116 int again;117};118 119struct rxe_comp_info {120 u32 psn;121 int opcode;122 int timeout;123 int timeout_retry;124 int started_retry;125 u32 retry_cnt;126 u32 rnr_retry;127};128 129enum rdatm_res_state {130 rdatm_res_state_next,131 rdatm_res_state_new,132 rdatm_res_state_replay,133};134 135struct resp_res {136 int type;137 int replay;138 u32 first_psn;139 u32 last_psn;140 u32 cur_psn;141 enum rdatm_res_state state;142 143 union {144 struct {145 u64 orig_val;146 } atomic;147 struct {148 u64 va_org;149 u32 rkey;150 u32 length;151 u64 va;152 u32 resid;153 } read;154 struct {155 u32 length;156 u64 va;157 u8 type;158 u8 level;159 } flush;160 };161};162 163struct rxe_resp_info {164 u32 msn;165 u32 psn;166 u32 ack_psn;167 int opcode;168 int drop_msg;169 int goto_error;170 int sent_psn_nak;171 enum ib_wc_status status;172 u8 aeth_syndrome;173 174 /* Receive only */175 struct rxe_recv_wqe *wqe;176 177 /* RDMA read / atomic only */178 u64 va;179 u64 offset;180 struct rxe_mr *mr;181 u32 resid;182 u32 rkey;183 u32 length;184 185 /* SRQ only */186 struct {187 struct rxe_recv_wqe wqe;188 struct ib_sge sge[RXE_MAX_SGE];189 } srq_wqe;190 191 /* Responder resources. It's a circular list where the oldest192 * resource is dropped first.193 */194 struct resp_res *resources;195 unsigned int res_head;196 unsigned int res_tail;197 struct resp_res *res;198};199 200struct rxe_qp {201 struct ib_qp ibqp;202 struct rxe_pool_elem elem;203 struct ib_qp_attr attr;204 unsigned int valid;205 unsigned int mtu;206 bool is_user;207 208 struct rxe_pd *pd;209 struct rxe_srq *srq;210 struct rxe_cq *scq;211 struct rxe_cq *rcq;212 213 enum ib_sig_type sq_sig_type;214 215 struct rxe_sq sq;216 struct rxe_rq rq;217 218 struct socket *sk;219 u32 dst_cookie;220 u16 src_port;221 222 struct rxe_av pri_av;223 struct rxe_av alt_av;224 225 atomic_t mcg_num;226 227 struct sk_buff_head req_pkts;228 struct sk_buff_head resp_pkts;229 230 struct rxe_task send_task;231 struct rxe_task recv_task;232 233 struct rxe_req_info req;234 struct rxe_comp_info comp;235 struct rxe_resp_info resp;236 237 atomic_t ssn;238 atomic_t skb_out;239 int need_req_skb;240 241 /* Timer for retranmitting packet when ACKs have been lost. RC242 * only. The requester sets it when it is not already243 * started. The responder resets it whenever an ack is244 * received.245 */246 struct timer_list retrans_timer;247 u64 qp_timeout_jiffies;248 249 /* Timer for handling RNR NAKS. */250 struct timer_list rnr_nak_timer;251 252 spinlock_t state_lock; /* guard requester and completer */253 254 struct execute_work cleanup_work;255};256 257enum {258 RXE_ACCESS_REMOTE = IB_ACCESS_REMOTE_READ259 | IB_ACCESS_REMOTE_WRITE260 | IB_ACCESS_REMOTE_ATOMIC,261 RXE_ACCESS_SUPPORTED_MR = RXE_ACCESS_REMOTE262 | IB_ACCESS_LOCAL_WRITE263 | IB_ACCESS_MW_BIND264 | IB_ACCESS_ON_DEMAND265 | IB_ACCESS_FLUSH_GLOBAL266 | IB_ACCESS_FLUSH_PERSISTENT267 | IB_ACCESS_OPTIONAL,268 RXE_ACCESS_SUPPORTED_QP = RXE_ACCESS_SUPPORTED_MR,269 RXE_ACCESS_SUPPORTED_MW = RXE_ACCESS_SUPPORTED_MR270 | IB_ZERO_BASED,271};272 273enum rxe_mr_state {274 RXE_MR_STATE_INVALID,275 RXE_MR_STATE_FREE,276 RXE_MR_STATE_VALID,277};278 279enum rxe_mr_copy_dir {280 RXE_TO_MR_OBJ,281 RXE_FROM_MR_OBJ,282};283 284enum rxe_mr_lookup_type {285 RXE_LOOKUP_LOCAL,286 RXE_LOOKUP_REMOTE,287};288 289enum rxe_rereg {290 RXE_MR_REREG_SUPPORTED = IB_MR_REREG_PD291 | IB_MR_REREG_ACCESS,292};293 294static inline int rkey_is_mw(u32 rkey)295{296 u32 index = rkey >> 8;297 298 return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX);299}300 301struct rxe_mr {302 struct rxe_pool_elem elem;303 struct ib_mr ibmr;304 305 struct ib_umem *umem;306 307 u32 lkey;308 u32 rkey;309 enum rxe_mr_state state;310 int access;311 atomic_t num_mw;312 313 unsigned int page_offset;314 unsigned int page_shift;315 u64 page_mask;316 317 u32 num_buf;318 u32 nbuf;319 320 struct xarray page_list;321};322 323static inline unsigned int mr_page_size(struct rxe_mr *mr)324{325 return mr ? mr->ibmr.page_size : PAGE_SIZE;326}327 328enum rxe_mw_state {329 RXE_MW_STATE_INVALID = RXE_MR_STATE_INVALID,330 RXE_MW_STATE_FREE = RXE_MR_STATE_FREE,331 RXE_MW_STATE_VALID = RXE_MR_STATE_VALID,332};333 334struct rxe_mw {335 struct ib_mw ibmw;336 struct rxe_pool_elem elem;337 spinlock_t lock;338 enum rxe_mw_state state;339 struct rxe_qp *qp; /* Type 2 only */340 struct rxe_mr *mr;341 u32 rkey;342 int access;343 u64 addr;344 u64 length;345};346 347struct rxe_mcg {348 struct rb_node node;349 struct kref ref_cnt;350 struct rxe_dev *rxe;351 struct list_head qp_list;352 union ib_gid mgid;353 atomic_t qp_num;354 u32 qkey;355 u16 pkey;356};357 358struct rxe_mca {359 struct list_head qp_list;360 struct rxe_qp *qp;361};362 363struct rxe_port {364 struct ib_port_attr attr;365 __be64 port_guid;366 __be64 subnet_prefix;367 spinlock_t port_lock; /* guard port */368 unsigned int mtu_cap;369 /* special QPs */370 u32 qp_gsi_index;371};372 373struct rxe_dev {374 struct ib_device ib_dev;375 struct ib_device_attr attr;376 int max_ucontext;377 int max_inline_data;378 struct mutex usdev_lock;379 380 struct net_device *ndev;381 382 struct rxe_pool uc_pool;383 struct rxe_pool pd_pool;384 struct rxe_pool ah_pool;385 struct rxe_pool srq_pool;386 struct rxe_pool qp_pool;387 struct rxe_pool cq_pool;388 struct rxe_pool mr_pool;389 struct rxe_pool mw_pool;390 391 /* multicast support */392 spinlock_t mcg_lock;393 struct rb_root mcg_tree;394 atomic_t mcg_num;395 atomic_t mcg_attach;396 397 spinlock_t pending_lock; /* guard pending_mmaps */398 struct list_head pending_mmaps;399 400 spinlock_t mmap_offset_lock; /* guard mmap_offset */401 u64 mmap_offset;402 403 atomic64_t stats_counters[RXE_NUM_OF_COUNTERS];404 405 struct rxe_port port;406 struct crypto_shash *tfm;407};408 409static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)410{411 atomic64_inc(&rxe->stats_counters[index]);412}413 414static inline struct rxe_dev *to_rdev(struct ib_device *dev)415{416 return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;417}418 419static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc)420{421 return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL;422}423 424static inline struct rxe_pd *to_rpd(struct ib_pd *pd)425{426 return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL;427}428 429static inline struct rxe_ah *to_rah(struct ib_ah *ah)430{431 return ah ? container_of(ah, struct rxe_ah, ibah) : NULL;432}433 434static inline struct rxe_srq *to_rsrq(struct ib_srq *srq)435{436 return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL;437}438 439static inline struct rxe_qp *to_rqp(struct ib_qp *qp)440{441 return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL;442}443 444static inline struct rxe_cq *to_rcq(struct ib_cq *cq)445{446 return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL;447}448 449static inline struct rxe_mr *to_rmr(struct ib_mr *mr)450{451 return mr ? container_of(mr, struct rxe_mr, ibmr) : NULL;452}453 454static inline struct rxe_mw *to_rmw(struct ib_mw *mw)455{456 return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL;457}458 459static inline struct rxe_pd *rxe_ah_pd(struct rxe_ah *ah)460{461 return to_rpd(ah->ibah.pd);462}463 464static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)465{466 return to_rpd(mr->ibmr.pd);467}468 469static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)470{471 return to_rpd(mw->ibmw.pd);472}473 474int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);475 476#endif /* RXE_VERBS_H */477