484 lines · c
1/* SPDX-License-Identifier: GPL-2.0-only */2/*3 * Copyright (c) 2022 Microsoft Corporation. All rights reserved.4 */5 6#ifndef _MANA_IB_H_7#define _MANA_IB_H_8 9#include <rdma/ib_verbs.h>10#include <rdma/ib_mad.h>11#include <rdma/ib_umem.h>12#include <rdma/mana-abi.h>13#include <rdma/uverbs_ioctl.h>14 15#include <net/mana/mana.h>16 17#define PAGE_SZ_BM \18 (SZ_4K | SZ_8K | SZ_16K | SZ_32K | SZ_64K | SZ_128K | SZ_256K | \19 SZ_512K | SZ_1M | SZ_2M)20 21/* MANA doesn't have any limit for MR size */22#define MANA_IB_MAX_MR_SIZE U64_MAX23 24/*25 * The hardware limit of number of MRs is greater than maximum number of MRs26 * that can possibly represent in 24 bits27 */28#define MANA_IB_MAX_MR 0xFFFFFFu29 30/*31 * The CA timeout is approx. 260ms (4us * 2^(DELAY))32 */33#define MANA_CA_ACK_DELAY 1634 35struct mana_ib_adapter_caps {36 u32 max_sq_id;37 u32 max_rq_id;38 u32 max_cq_id;39 u32 max_qp_count;40 u32 max_cq_count;41 u32 max_mr_count;42 u32 max_pd_count;43 u32 max_inbound_read_limit;44 u32 max_outbound_read_limit;45 u32 mw_count;46 u32 max_srq_count;47 u32 max_qp_wr;48 u32 max_send_sge_count;49 u32 max_recv_sge_count;50 u32 max_inline_data_size;51};52 53struct mana_ib_queue {54 struct ib_umem *umem;55 u64 gdma_region;56 u64 id;57};58 59struct mana_ib_dev {60 struct ib_device ib_dev;61 struct gdma_dev *gdma_dev;62 mana_handle_t adapter_handle;63 struct gdma_queue *fatal_err_eq;64 struct gdma_queue **eqs;65 struct xarray qp_table_wq;66 struct mana_ib_adapter_caps adapter_caps;67};68 69struct mana_ib_wq {70 struct ib_wq ibwq;71 struct mana_ib_queue queue;72 int wqe;73 u32 wq_buf_size;74 mana_handle_t rx_object;75};76 77struct mana_ib_pd {78 struct ib_pd ibpd;79 u32 pdn;80 mana_handle_t pd_handle;81 82 /* Mutex for sharing access to vport_use_count */83 struct mutex vport_mutex;84 int vport_use_count;85 86 bool tx_shortform_allowed;87 u32 tx_vp_offset;88};89 90struct mana_ib_mr {91 struct ib_mr ibmr;92 struct ib_umem *umem;93 mana_handle_t mr_handle;94};95 96struct mana_ib_cq {97 struct ib_cq ibcq;98 struct mana_ib_queue queue;99 int cqe;100 u32 comp_vector;101 mana_handle_t cq_handle;102};103 104enum mana_rc_queue_type {105 MANA_RC_SEND_QUEUE_REQUESTER = 0,106 MANA_RC_SEND_QUEUE_RESPONDER,107 MANA_RC_SEND_QUEUE_FMR,108 MANA_RC_RECV_QUEUE_REQUESTER,109 MANA_RC_RECV_QUEUE_RESPONDER,110 MANA_RC_QUEUE_TYPE_MAX,111};112 113struct mana_ib_rc_qp {114 struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX];115};116 117struct mana_ib_qp {118 struct ib_qp ibqp;119 120 mana_handle_t qp_handle;121 union {122 struct mana_ib_queue raw_sq;123 struct mana_ib_rc_qp rc_qp;124 };125 126 /* The port on the IB device, starting with 1 */127 u32 port;128 129 refcount_t refcount;130 struct completion free;131};132 133struct mana_ib_ucontext {134 struct ib_ucontext ibucontext;135 u32 doorbell;136};137 138struct mana_ib_rwq_ind_table {139 struct ib_rwq_ind_table ib_ind_table;140};141 142enum mana_ib_command_code {143 MANA_IB_GET_ADAPTER_CAP = 0x30001,144 MANA_IB_CREATE_ADAPTER = 0x30002,145 MANA_IB_DESTROY_ADAPTER = 0x30003,146 MANA_IB_CONFIG_IP_ADDR = 0x30004,147 MANA_IB_CONFIG_MAC_ADDR = 0x30005,148 MANA_IB_CREATE_CQ = 0x30008,149 MANA_IB_DESTROY_CQ = 0x30009,150 MANA_IB_CREATE_RC_QP = 0x3000a,151 MANA_IB_DESTROY_RC_QP = 0x3000b,152 MANA_IB_SET_QP_STATE = 0x3000d,153};154 155struct mana_ib_query_adapter_caps_req {156 struct gdma_req_hdr hdr;157}; /*HW Data */158 159struct mana_ib_query_adapter_caps_resp {160 struct gdma_resp_hdr hdr;161 u32 max_sq_id;162 u32 max_rq_id;163 u32 max_cq_id;164 u32 max_qp_count;165 u32 max_cq_count;166 u32 max_mr_count;167 u32 max_pd_count;168 u32 max_inbound_read_limit;169 u32 max_outbound_read_limit;170 u32 mw_count;171 u32 max_srq_count;172 u32 max_requester_sq_size;173 u32 max_responder_sq_size;174 u32 max_requester_rq_size;175 u32 max_responder_rq_size;176 u32 max_send_sge_count;177 u32 max_recv_sge_count;178 u32 max_inline_data_size;179}; /* HW Data */180 181struct mana_rnic_create_adapter_req {182 struct gdma_req_hdr hdr;183 u32 notify_eq_id;184 u32 reserved;185 u64 feature_flags;186}; /*HW Data */187 188struct mana_rnic_create_adapter_resp {189 struct gdma_resp_hdr hdr;190 mana_handle_t adapter;191}; /* HW Data */192 193struct mana_rnic_destroy_adapter_req {194 struct gdma_req_hdr hdr;195 mana_handle_t adapter;196}; /*HW Data */197 198struct mana_rnic_destroy_adapter_resp {199 struct gdma_resp_hdr hdr;200}; /* HW Data */201 202enum mana_ib_addr_op {203 ADDR_OP_ADD = 1,204 ADDR_OP_REMOVE = 2,205};206 207enum sgid_entry_type {208 SGID_TYPE_IPV4 = 1,209 SGID_TYPE_IPV6 = 2,210};211 212struct mana_rnic_config_addr_req {213 struct gdma_req_hdr hdr;214 mana_handle_t adapter;215 enum mana_ib_addr_op op;216 enum sgid_entry_type sgid_type;217 u8 ip_addr[16];218}; /* HW Data */219 220struct mana_rnic_config_addr_resp {221 struct gdma_resp_hdr hdr;222}; /* HW Data */223 224struct mana_rnic_config_mac_addr_req {225 struct gdma_req_hdr hdr;226 mana_handle_t adapter;227 enum mana_ib_addr_op op;228 u8 mac_addr[ETH_ALEN];229 u8 reserved[6];230}; /* HW Data */231 232struct mana_rnic_config_mac_addr_resp {233 struct gdma_resp_hdr hdr;234}; /* HW Data */235 236struct mana_rnic_create_cq_req {237 struct gdma_req_hdr hdr;238 mana_handle_t adapter;239 u64 gdma_region;240 u32 eq_id;241 u32 doorbell_page;242}; /* HW Data */243 244struct mana_rnic_create_cq_resp {245 struct gdma_resp_hdr hdr;246 mana_handle_t cq_handle;247 u32 cq_id;248 u32 reserved;249}; /* HW Data */250 251struct mana_rnic_destroy_cq_req {252 struct gdma_req_hdr hdr;253 mana_handle_t adapter;254 mana_handle_t cq_handle;255}; /* HW Data */256 257struct mana_rnic_destroy_cq_resp {258 struct gdma_resp_hdr hdr;259}; /* HW Data */260 261enum mana_rnic_create_rc_flags {262 MANA_RC_FLAG_NO_FMR = 2,263};264 265struct mana_rnic_create_qp_req {266 struct gdma_req_hdr hdr;267 mana_handle_t adapter;268 mana_handle_t pd_handle;269 mana_handle_t send_cq_handle;270 mana_handle_t recv_cq_handle;271 u64 dma_region[MANA_RC_QUEUE_TYPE_MAX];272 u64 deprecated[2];273 u64 flags;274 u32 doorbell_page;275 u32 max_send_wr;276 u32 max_recv_wr;277 u32 max_send_sge;278 u32 max_recv_sge;279 u32 reserved;280}; /* HW Data */281 282struct mana_rnic_create_qp_resp {283 struct gdma_resp_hdr hdr;284 mana_handle_t rc_qp_handle;285 u32 queue_ids[MANA_RC_QUEUE_TYPE_MAX];286 u32 reserved;287}; /* HW Data*/288 289struct mana_rnic_destroy_rc_qp_req {290 struct gdma_req_hdr hdr;291 mana_handle_t adapter;292 mana_handle_t rc_qp_handle;293}; /* HW Data */294 295struct mana_rnic_destroy_rc_qp_resp {296 struct gdma_resp_hdr hdr;297}; /* HW Data */298 299struct mana_ib_ah_attr {300 u8 src_addr[16];301 u8 dest_addr[16];302 u8 src_mac[ETH_ALEN];303 u8 dest_mac[ETH_ALEN];304 u8 src_addr_type;305 u8 dest_addr_type;306 u8 hop_limit;307 u8 traffic_class;308 u16 src_port;309 u16 dest_port;310 u32 reserved;311};312 313struct mana_rnic_set_qp_state_req {314 struct gdma_req_hdr hdr;315 mana_handle_t adapter;316 mana_handle_t qp_handle;317 u64 attr_mask;318 u32 qp_state;319 u32 path_mtu;320 u32 rq_psn;321 u32 sq_psn;322 u32 dest_qpn;323 u32 max_dest_rd_atomic;324 u32 retry_cnt;325 u32 rnr_retry;326 u32 min_rnr_timer;327 u32 reserved;328 struct mana_ib_ah_attr ah_attr;329}; /* HW Data */330 331struct mana_rnic_set_qp_state_resp {332 struct gdma_resp_hdr hdr;333}; /* HW Data */334 335static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)336{337 return mdev->gdma_dev->gdma_context;338}339 340static inline struct mana_ib_qp *mana_get_qp_ref(struct mana_ib_dev *mdev,341 uint32_t qid)342{343 struct mana_ib_qp *qp;344 unsigned long flag;345 346 xa_lock_irqsave(&mdev->qp_table_wq, flag);347 qp = xa_load(&mdev->qp_table_wq, qid);348 if (qp)349 refcount_inc(&qp->refcount);350 xa_unlock_irqrestore(&mdev->qp_table_wq, flag);351 return qp;352}353 354static inline void mana_put_qp_ref(struct mana_ib_qp *qp)355{356 if (refcount_dec_and_test(&qp->refcount))357 complete(&qp->free);358}359 360static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 port)361{362 struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);363 struct gdma_context *gc = mdev_to_gc(mdev);364 struct mana_context *mc = gc->mana.driver_data;365 366 if (port < 1 || port > mc->num_ports)367 return NULL;368 return mc->ports[port - 1];369}370 371static inline void copy_in_reverse(u8 *dst, const u8 *src, u32 size)372{373 u32 i;374 375 for (i = 0; i < size; i++)376 dst[size - 1 - i] = src[i];377}378 379int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);380void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);381 382int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,383 mana_handle_t *gdma_region);384 385int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,386 mana_handle_t *gdma_region, u64 virt);387 388int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,389 mana_handle_t gdma_region);390 391int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,392 struct mana_ib_queue *queue);393void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue);394 395struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,396 struct ib_wq_init_attr *init_attr,397 struct ib_udata *udata);398 399int mana_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,400 u32 wq_attr_mask, struct ib_udata *udata);401 402int mana_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata);403 404int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,405 struct ib_rwq_ind_table_init_attr *init_attr,406 struct ib_udata *udata);407 408int mana_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl);409 410struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags);411 412struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,413 u64 iova, int access_flags,414 struct ib_udata *udata);415 416int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);417 418int mana_ib_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr,419 struct ib_udata *udata);420 421int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,422 int attr_mask, struct ib_udata *udata);423 424int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);425 426int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port_id,427 struct mana_ib_pd *pd, u32 doorbell_id);428void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,429 u32 port);430 431int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,432 struct uverbs_attr_bundle *attrs);433 434int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);435 436int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);437int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);438 439int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,440 struct ib_udata *udata);441void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext);442 443int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma);444 445int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,446 struct ib_port_immutable *immutable);447int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,448 struct ib_udata *uhw);449int mana_ib_query_port(struct ib_device *ibdev, u32 port,450 struct ib_port_attr *props);451int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,452 union ib_gid *gid);453 454void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext);455 456int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev);457 458int mana_ib_create_eqs(struct mana_ib_dev *mdev);459 460void mana_ib_destroy_eqs(struct mana_ib_dev *mdev);461 462int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev);463 464int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev);465 466int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey);467 468enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num);469 470int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context);471 472int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context);473 474int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac);475 476int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell);477 478int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);479 480int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,481 struct ib_qp_init_attr *attr, u32 doorbell, u64 flags);482int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp);483#endif484