608 lines · c
1/* This file is part of the Emulex RoCE Device Driver for2 * RoCE (RDMA over Converged Ethernet) adapters.3 * Copyright (C) 2012-2015 Emulex. All rights reserved.4 * EMULEX and SLI are trademarks of Emulex.5 * www.emulex.com6 *7 * This software is available to you under a choice of one of two licenses.8 * You may choose to be licensed under the terms of the GNU General Public9 * License (GPL) Version 2, available from the file COPYING in the main10 * directory of this source tree, or the BSD license below:11 *12 * Redistribution and use in source and binary forms, with or without13 * modification, are permitted provided that the following conditions14 * are met:15 *16 * - Redistributions of source code must retain the above copyright notice,17 * this list of conditions and the following disclaimer.18 *19 * - Redistributions in binary form must reproduce the above copyright20 * notice, this list of conditions and the following disclaimer in21 * the documentation and/or other materials provided with the distribution.22 *23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR30 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,31 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR32 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF33 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.34 *35 * Contact Information:36 * linux-drivers@emulex.com37 *38 * Emulex39 * 3333 Susan Street40 * Costa Mesa, CA 9262641 */42 43#ifndef __OCRDMA_H__44#define __OCRDMA_H__45 46#include <linux/mutex.h>47#include <linux/list.h>48#include <linux/spinlock.h>49#include <linux/pci.h>50 51#include <rdma/ib_verbs.h>52#include <rdma/ib_user_verbs.h>53#include <rdma/ib_addr.h>54 55#include <be_roce.h>56#include "ocrdma_sli.h"57 58#define OCRDMA_ROCE_DRV_VERSION "11.0.0.0"59 60#define OCRDMA_ROCE_DRV_DESC "Emulex OneConnect RoCE Driver"61#define OCRDMA_NODE_DESC "Emulex OneConnect RoCE HCA"62 63#define OC_NAME_SH OCRDMA_NODE_DESC "(Skyhawk)"64#define OC_NAME_UNKNOWN OCRDMA_NODE_DESC "(Unknown)"65 66#define OC_SKH_DEVICE_PF 0x72067#define OC_SKH_DEVICE_VF 0x72868#define OCRDMA_MAX_AH 51269 70#define OCRDMA_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME)71 72#define convert_to_64bit(lo, hi) ((u64)hi << 32 | (u64)lo)73#define EQ_INTR_PER_SEC_THRSH_HI 15000074#define EQ_INTR_PER_SEC_THRSH_LOW 10000075#define EQ_AIC_MAX_EQD 2076#define EQ_AIC_MIN_EQD 077 78void ocrdma_eqd_set_task(struct work_struct *work);79 80struct ocrdma_dev_attr {81 u8 fw_ver[32];82 u32 vendor_id;83 u32 device_id;84 u16 max_pd;85 u16 max_dpp_pds;86 u16 max_cq;87 u16 max_cqe;88 u16 max_qp;89 u16 max_wqe;90 u16 max_rqe;91 u16 max_srq;92 u32 max_inline_data;93 int max_send_sge;94 int max_recv_sge;95 int max_srq_sge;96 int max_rdma_sge;97 int max_mr;98 u64 max_mr_size;99 u32 max_num_mr_pbl;100 int max_mw;101 int max_map_per_fmr;102 int max_pages_per_frmr;103 u16 max_ord_per_qp;104 u16 max_ird_per_qp;105 106 int device_cap_flags;107 u8 cq_overflow_detect;108 u8 srq_supported;109 110 u32 wqe_size;111 u32 rqe_size;112 u32 ird_page_size;113 u8 local_ca_ack_delay;114 u8 ird;115 u8 num_ird_pages;116 u8 udp_encap;117};118 119struct ocrdma_dma_mem {120 void *va;121 dma_addr_t pa;122 u32 size;123};124 125struct ocrdma_pbl {126 void *va;127 dma_addr_t pa;128};129 130struct ocrdma_queue_info {131 void *va;132 dma_addr_t dma;133 u32 size;134 u16 len;135 u16 entry_size; /* Size of an element in the queue */136 u16 id; /* qid, where to ring the doorbell. */137 u16 head, tail;138 bool created;139};140 141struct ocrdma_aic_obj { /* Adaptive interrupt coalescing (AIC) info */142 u32 prev_eqd;143 u64 eq_intr_cnt;144 u64 prev_eq_intr_cnt;145};146 147struct ocrdma_eq {148 struct ocrdma_queue_info q;149 u32 vector;150 int cq_cnt;151 struct ocrdma_dev *dev;152 char irq_name[32];153 struct ocrdma_aic_obj aic_obj;154};155 156struct ocrdma_mq {157 struct ocrdma_queue_info sq;158 struct ocrdma_queue_info cq;159 bool rearm_cq;160};161 162struct mqe_ctx {163 struct mutex lock; /* for serializing mailbox commands on MQ */164 wait_queue_head_t cmd_wait;165 u32 tag;166 u16 cqe_status;167 u16 ext_status;168 bool cmd_done;169 bool fw_error_state;170};171 172struct ocrdma_hw_mr {173 u32 lkey;174 u8 fr_mr;175 u8 remote_atomic;176 u8 remote_rd;177 u8 remote_wr;178 u8 local_rd;179 u8 local_wr;180 u8 mw_bind;181 u8 rsvd;182 u64 len;183 struct ocrdma_pbl *pbl_table;184 u32 num_pbls;185 u32 num_pbes;186 u32 pbl_size;187 u32 pbe_size;188 u64 va;189};190 191struct ocrdma_mr {192 struct ib_mr ibmr;193 struct ib_umem *umem;194 struct ocrdma_hw_mr hwmr;195 u64 *pages;196 u32 npages;197};198 199struct ocrdma_stats {200 u8 type;201 struct ocrdma_dev *dev;202};203 204struct ocrdma_pd_resource_mgr {205 u32 pd_norm_start;206 u16 pd_norm_count;207 u16 pd_norm_thrsh;208 u16 max_normal_pd;209 u32 pd_dpp_start;210 u16 pd_dpp_count;211 u16 pd_dpp_thrsh;212 u16 max_dpp_pd;213 u16 dpp_page_index;214 unsigned long *pd_norm_bitmap;215 unsigned long *pd_dpp_bitmap;216 bool pd_prealloc_valid;217};218 219struct stats_mem {220 struct ocrdma_mqe mqe;221 void *va;222 dma_addr_t pa;223 u32 size;224 char *debugfs_mem;225};226 227struct phy_info {228 u16 auto_speeds_supported;229 u16 fixed_speeds_supported;230 u16 phy_type;231 u16 interface_type;232};233 234enum ocrdma_flags {235 OCRDMA_FLAGS_LINK_STATUS_INIT = 0x01236};237 238struct ocrdma_dev {239 struct ib_device ibdev;240 struct ocrdma_dev_attr attr;241 242 struct mutex dev_lock; /* provides syncronise access to device data */243 spinlock_t flush_q_lock ____cacheline_aligned;244 245 struct ocrdma_cq **cq_tbl;246 struct ocrdma_qp **qp_tbl;247 248 struct ocrdma_eq *eq_tbl;249 int eq_cnt;250 struct delayed_work eqd_work;251 u16 base_eqid;252 u16 max_eq;253 254 /* provided synchronization to sgid table for255 * updating gid entries triggered by notifier.256 */257 spinlock_t sgid_lock;258 259 int gsi_qp_created;260 struct ocrdma_cq *gsi_sqcq;261 struct ocrdma_cq *gsi_rqcq;262 263 struct {264 struct ocrdma_av *va;265 dma_addr_t pa;266 u32 size;267 u32 num_ah;268 /* provide synchronization for av269 * entry allocations.270 */271 spinlock_t lock;272 u32 ahid;273 struct ocrdma_pbl pbl;274 } av_tbl;275 276 void *mbx_cmd;277 struct ocrdma_mq mq;278 struct mqe_ctx mqe_ctx;279 280 struct be_dev_info nic_info;281 struct phy_info phy;282 char model_number[32];283 u32 hba_port_num;284 285 struct list_head entry;286 int id;287 u64 *stag_arr;288 u8 sl; /* service level */289 bool pfc_state;290 atomic_t update_sl;291 u16 pvid;292 u32 asic_id;293 u32 flags;294 295 ulong last_stats_time;296 struct mutex stats_lock; /* provide synch for debugfs operations */297 struct stats_mem stats_mem;298 struct ocrdma_stats rsrc_stats;299 struct ocrdma_stats rx_stats;300 struct ocrdma_stats wqe_stats;301 struct ocrdma_stats tx_stats;302 struct ocrdma_stats db_err_stats;303 struct ocrdma_stats tx_qp_err_stats;304 struct ocrdma_stats rx_qp_err_stats;305 struct ocrdma_stats tx_dbg_stats;306 struct ocrdma_stats rx_dbg_stats;307 struct ocrdma_stats driver_stats;308 struct ocrdma_stats reset_stats;309 struct dentry *dir;310 atomic_t async_err_stats[OCRDMA_MAX_ASYNC_ERRORS];311 atomic_t cqe_err_stats[OCRDMA_MAX_CQE_ERR];312 struct ocrdma_pd_resource_mgr *pd_mgr;313};314 315struct ocrdma_cq {316 struct ib_cq ibcq;317 struct ocrdma_cqe *va;318 u32 phase;319 u32 getp; /* pointer to pending wrs to320 * return to stack, wrap arounds321 * at max_hw_cqe322 */323 u32 max_hw_cqe;324 bool phase_change;325 spinlock_t cq_lock ____cacheline_aligned; /* provide synchronization326 * to cq polling327 */328 /* syncronizes cq completion handler invoked from multiple context */329 spinlock_t comp_handler_lock ____cacheline_aligned;330 u16 id;331 u16 eqn;332 333 struct ocrdma_ucontext *ucontext;334 dma_addr_t pa;335 u32 len;336 u32 cqe_cnt;337 338 /* head of all qp's sq and rq for which cqes need to be flushed339 * by the software.340 */341 struct list_head sq_head, rq_head;342};343 344struct ocrdma_pd {345 struct ib_pd ibpd;346 struct ocrdma_ucontext *uctx;347 u32 id;348 int num_dpp_qp;349 u32 dpp_page;350 bool dpp_enabled;351};352 353struct ocrdma_ah {354 struct ib_ah ibah;355 struct ocrdma_av *av;356 u16 sgid_index;357 u32 id;358 u8 hdr_type;359};360 361struct ocrdma_qp_hwq_info {362 u8 *va; /* virtual address */363 u32 max_sges;364 u32 head, tail;365 u32 entry_size;366 u32 max_cnt;367 u32 max_wqe_idx;368 u16 dbid; /* qid, where to ring the doorbell. */369 u32 len;370 dma_addr_t pa;371};372 373struct ocrdma_srq {374 struct ib_srq ibsrq;375 u8 __iomem *db;376 struct ocrdma_qp_hwq_info rq;377 u64 *rqe_wr_id_tbl;378 u32 *idx_bit_fields;379 u32 bit_fields_len;380 381 /* provide synchronization to multiple context(s) posting rqe */382 spinlock_t q_lock ____cacheline_aligned;383 384 struct ocrdma_pd *pd;385 u32 id;386};387 388struct ocrdma_qp {389 struct ib_qp ibqp;390 391 u8 __iomem *sq_db;392 struct ocrdma_qp_hwq_info sq;393 struct {394 uint64_t wrid;395 uint16_t dpp_wqe_idx;396 uint16_t dpp_wqe;397 uint8_t signaled;398 uint8_t rsvd[3];399 } *wqe_wr_id_tbl;400 u32 max_inline_data;401 402 /* provide synchronization to multiple context(s) posting wqe, rqe */403 spinlock_t q_lock ____cacheline_aligned;404 struct ocrdma_cq *sq_cq;405 /* list maintained per CQ to flush SQ errors */406 struct list_head sq_entry;407 408 u8 __iomem *rq_db;409 struct ocrdma_qp_hwq_info rq;410 u64 *rqe_wr_id_tbl;411 struct ocrdma_cq *rq_cq;412 struct ocrdma_srq *srq;413 /* list maintained per CQ to flush RQ errors */414 struct list_head rq_entry;415 416 enum ocrdma_qp_state state; /* QP state */417 int cap_flags;418 u32 max_ord, max_ird;419 420 u32 id;421 struct ocrdma_pd *pd;422 423 enum ib_qp_type qp_type;424 425 int sgid_idx;426 u32 qkey;427 bool dpp_enabled;428 u8 *ird_q_va;429 bool signaled;430};431 432struct ocrdma_ucontext {433 struct ib_ucontext ibucontext;434 435 struct list_head mm_head;436 struct mutex mm_list_lock; /* protects list entries of mm type */437 struct ocrdma_pd *cntxt_pd;438 int pd_in_use;439 440 struct {441 u32 *va;442 dma_addr_t pa;443 u32 len;444 } ah_tbl;445};446 447struct ocrdma_mm {448 struct {449 u64 phy_addr;450 unsigned long len;451 } key;452 struct list_head entry;453};454 455static inline struct ocrdma_dev *get_ocrdma_dev(struct ib_device *ibdev)456{457 return container_of(ibdev, struct ocrdma_dev, ibdev);458}459 460static inline struct ocrdma_ucontext *get_ocrdma_ucontext(struct ib_ucontext461 *ibucontext)462{463 return container_of(ibucontext, struct ocrdma_ucontext, ibucontext);464}465 466static inline struct ocrdma_pd *get_ocrdma_pd(struct ib_pd *ibpd)467{468 return container_of(ibpd, struct ocrdma_pd, ibpd);469}470 471static inline struct ocrdma_cq *get_ocrdma_cq(struct ib_cq *ibcq)472{473 return container_of(ibcq, struct ocrdma_cq, ibcq);474}475 476static inline struct ocrdma_qp *get_ocrdma_qp(struct ib_qp *ibqp)477{478 return container_of(ibqp, struct ocrdma_qp, ibqp);479}480 481static inline struct ocrdma_mr *get_ocrdma_mr(struct ib_mr *ibmr)482{483 return container_of(ibmr, struct ocrdma_mr, ibmr);484}485 486static inline struct ocrdma_ah *get_ocrdma_ah(struct ib_ah *ibah)487{488 return container_of(ibah, struct ocrdma_ah, ibah);489}490 491static inline struct ocrdma_srq *get_ocrdma_srq(struct ib_srq *ibsrq)492{493 return container_of(ibsrq, struct ocrdma_srq, ibsrq);494}495 496static inline int is_cqe_valid(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe)497{498 int cqe_valid;499 cqe_valid = le32_to_cpu(cqe->flags_status_srcqpn) & OCRDMA_CQE_VALID;500 return (cqe_valid == cq->phase);501}502 503static inline int is_cqe_for_sq(struct ocrdma_cqe *cqe)504{505 return (le32_to_cpu(cqe->flags_status_srcqpn) &506 OCRDMA_CQE_QTYPE) ? 0 : 1;507}508 509static inline int is_cqe_invalidated(struct ocrdma_cqe *cqe)510{511 return (le32_to_cpu(cqe->flags_status_srcqpn) &512 OCRDMA_CQE_INVALIDATE) ? 1 : 0;513}514 515static inline int is_cqe_imm(struct ocrdma_cqe *cqe)516{517 return (le32_to_cpu(cqe->flags_status_srcqpn) &518 OCRDMA_CQE_IMM) ? 1 : 0;519}520 521static inline int is_cqe_wr_imm(struct ocrdma_cqe *cqe)522{523 return (le32_to_cpu(cqe->flags_status_srcqpn) &524 OCRDMA_CQE_WRITE_IMM) ? 1 : 0;525}526 527static inline int ocrdma_resolve_dmac(struct ocrdma_dev *dev,528 struct rdma_ah_attr *ah_attr, u8 *mac_addr)529{530 struct in6_addr in6;531 532 memcpy(&in6, rdma_ah_read_grh(ah_attr)->dgid.raw, sizeof(in6));533 if (rdma_is_multicast_addr(&in6))534 rdma_get_mcast_mac(&in6, mac_addr);535 else if (rdma_link_local_addr(&in6))536 rdma_get_ll_mac(&in6, mac_addr);537 else538 memcpy(mac_addr, ah_attr->roce.dmac, ETH_ALEN);539 return 0;540}541 542static inline char *hca_name(struct ocrdma_dev *dev)543{544 switch (dev->nic_info.pdev->device) {545 case OC_SKH_DEVICE_PF:546 case OC_SKH_DEVICE_VF:547 return OC_NAME_SH;548 default:549 return OC_NAME_UNKNOWN;550 }551}552 553static inline int ocrdma_get_eq_table_index(struct ocrdma_dev *dev,554 int eqid)555{556 int indx;557 558 for (indx = 0; indx < dev->eq_cnt; indx++) {559 if (dev->eq_tbl[indx].q.id == eqid)560 return indx;561 }562 563 return -EINVAL;564}565 566static inline u8 ocrdma_get_asic_type(struct ocrdma_dev *dev)567{568 if (dev->nic_info.dev_family == 0xF && !dev->asic_id) {569 pci_read_config_dword(570 dev->nic_info.pdev,571 OCRDMA_SLI_ASIC_ID_OFFSET, &dev->asic_id);572 }573 574 return (dev->asic_id & OCRDMA_SLI_ASIC_GEN_NUM_MASK) >>575 OCRDMA_SLI_ASIC_GEN_NUM_SHIFT;576}577 578static inline u8 ocrdma_get_pfc_prio(u8 *pfc, u8 prio)579{580 return *(pfc + prio);581}582 583static inline u8 ocrdma_get_app_prio(u8 *app_prio, u8 prio)584{585 return *(app_prio + prio);586}587 588static inline u8 ocrdma_is_enabled_and_synced(u32 state)589{ /* May also be used to interpret TC-state, QCN-state590 * Appl-state and Logical-link-state in future.591 */592 return (state & OCRDMA_STATE_FLAG_ENABLED) &&593 (state & OCRDMA_STATE_FLAG_SYNC);594}595 596static inline u8 ocrdma_get_ae_link_state(u32 ae_state)597{598 return ((ae_state & OCRDMA_AE_LSC_LS_MASK) >> OCRDMA_AE_LSC_LS_SHIFT);599}600 601static inline bool ocrdma_is_udp_encap_supported(struct ocrdma_dev *dev)602{603 return (dev->attr.udp_encap & OCRDMA_L3_TYPE_IPV4) ||604 (dev->attr.udp_encap & OCRDMA_L3_TYPE_IPV6);605}606 607#endif608