1077 lines · c
1/*2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.3 *4 * This software is available to you under a choice of one of two5 * licenses. You may choose to be licensed under the terms of the GNU6 * General Public License (GPL) Version 2, available from the file7 * COPYING in the main directory of this source tree, or the8 * OpenIB.org BSD license below:9 *10 * Redistribution and use in source and binary forms, with or11 * without modification, are permitted provided that the following12 * conditions are met:13 *14 * - Redistributions of source code must retain the above15 * copyright notice, this list of conditions and the following16 * disclaimer.17 * - Redistributions in binary form must reproduce the above18 * copyright notice, this list of conditions and the following19 * disclaimer in the documentation and/or other materials20 * provided with the distribution.21 *22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,23 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND25 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS26 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN27 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN28 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE29 * SOFTWARE.30 */31#ifndef __IW_CXGB4_H__32#define __IW_CXGB4_H__33 34#include <linux/mutex.h>35#include <linux/list.h>36#include <linux/spinlock.h>37#include <linux/xarray.h>38#include <linux/completion.h>39#include <linux/netdevice.h>40#include <linux/sched/mm.h>41#include <linux/pci.h>42#include <linux/dma-mapping.h>43#include <linux/inet.h>44#include <linux/wait.h>45#include <linux/kref.h>46#include <linux/timer.h>47#include <linux/io.h>48#include <linux/workqueue.h>49 50#include <asm/byteorder.h>51 52#include <net/net_namespace.h>53 54#include <rdma/ib_verbs.h>55#include <rdma/iw_cm.h>56#include <rdma/rdma_netlink.h>57#include <rdma/iw_portmap.h>58#include <rdma/restrack.h>59 60#include "cxgb4.h"61#include "cxgb4_uld.h"62#include "l2t.h"63#include <rdma/cxgb4-abi.h>64 65#define DRV_NAME "iw_cxgb4"66#define MOD DRV_NAME ":"67 68#ifdef pr_fmt69#undef pr_fmt70#endif71 72#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt73 74#include "t4.h"75 76#define PBL_OFF(rdev_p, a) ((a) - (rdev_p)->lldi.vr->pbl.start)77#define RQT_OFF(rdev_p, a) ((a) - (rdev_p)->lldi.vr->rq.start)78 79static inline void *cplhdr(struct sk_buff *skb)80{81 return skb->data;82}83 84#define C4IW_ID_TABLE_F_RANDOM 1 /* Pseudo-randomize the id's returned */85#define C4IW_ID_TABLE_F_EMPTY 2 /* Table is initially empty */86 87struct c4iw_id_table {88 u32 flags;89 u32 start; /* logical minimal id */90 u32 last; /* hint for find */91 u32 max;92 spinlock_t lock;93 unsigned long *table;94};95 96struct c4iw_resource {97 struct c4iw_id_table tpt_table;98 struct c4iw_id_table qid_table;99 struct c4iw_id_table pdid_table;100 struct c4iw_id_table srq_table;101};102 103struct c4iw_qid_list {104 struct list_head entry;105 u32 qid;106};107 108struct c4iw_dev_ucontext {109 struct list_head qpids;110 struct list_head cqids;111 struct mutex lock;112 struct kref kref;113};114 115enum c4iw_rdev_flags {116 T4_FATAL_ERROR = (1<<0),117 T4_STATUS_PAGE_DISABLED = (1<<1),118};119 120struct c4iw_stat {121 u64 total;122 u64 cur;123 u64 max;124 u64 fail;125};126 127struct c4iw_stats {128 struct mutex lock;129 struct c4iw_stat qid;130 struct c4iw_stat pd;131 struct c4iw_stat stag;132 struct c4iw_stat pbl;133 struct c4iw_stat rqt;134 struct c4iw_stat srqt;135 struct c4iw_stat srq;136 struct c4iw_stat ocqp;137 u64 db_full;138 u64 db_empty;139 u64 db_drop;140 u64 db_state_transitions;141 u64 db_fc_interruptions;142 u64 tcam_full;143 u64 act_ofld_conn_fails;144 u64 pas_ofld_conn_fails;145 u64 neg_adv;146};147 148struct c4iw_hw_queue {149 int t4_eq_status_entries;150 int t4_max_eq_size;151 int t4_max_iq_size;152 int t4_max_rq_size;153 int t4_max_sq_size;154 int t4_max_qp_depth;155 int t4_max_cq_depth;156 int t4_stat_len;157};158 159struct wr_log_entry {160 ktime_t post_host_time;161 ktime_t poll_host_time;162 u64 post_sge_ts;163 u64 cqe_sge_ts;164 u64 poll_sge_ts;165 u16 qid;166 u16 wr_id;167 u8 opcode;168 u8 valid;169};170 171struct c4iw_rdev {172 struct c4iw_resource resource;173 u32 qpmask;174 u32 cqmask;175 struct c4iw_dev_ucontext uctx;176 struct gen_pool *pbl_pool;177 struct gen_pool *rqt_pool;178 struct gen_pool *ocqp_pool;179 u32 flags;180 struct cxgb4_lld_info lldi;181 unsigned long bar2_pa;182 void __iomem *bar2_kva;183 unsigned long oc_mw_pa;184 void __iomem *oc_mw_kva;185 struct c4iw_stats stats;186 struct c4iw_hw_queue hw_queue;187 struct t4_dev_status_page *status_page;188 atomic_t wr_log_idx;189 struct wr_log_entry *wr_log;190 int wr_log_size;191 struct workqueue_struct *free_workq;192 struct completion rqt_compl;193 struct completion pbl_compl;194 struct kref rqt_kref;195 struct kref pbl_kref;196};197 198static inline int c4iw_fatal_error(struct c4iw_rdev *rdev)199{200 return rdev->flags & T4_FATAL_ERROR;201}202 203static inline int c4iw_num_stags(struct c4iw_rdev *rdev)204{205 return (int)(rdev->lldi.vr->stag.size >> 5);206}207 208#define C4IW_WR_TO (60*HZ)209 210struct c4iw_wr_wait {211 struct completion completion;212 int ret;213 struct kref kref;214};215 216void _c4iw_free_wr_wait(struct kref *kref);217 218static inline void c4iw_put_wr_wait(struct c4iw_wr_wait *wr_waitp)219{220 pr_debug("wr_wait %p ref before put %u\n", wr_waitp,221 kref_read(&wr_waitp->kref));222 WARN_ON(kref_read(&wr_waitp->kref) == 0);223 kref_put(&wr_waitp->kref, _c4iw_free_wr_wait);224}225 226static inline void c4iw_get_wr_wait(struct c4iw_wr_wait *wr_waitp)227{228 pr_debug("wr_wait %p ref before get %u\n", wr_waitp,229 kref_read(&wr_waitp->kref));230 WARN_ON(kref_read(&wr_waitp->kref) == 0);231 kref_get(&wr_waitp->kref);232}233 234static inline void c4iw_init_wr_wait(struct c4iw_wr_wait *wr_waitp)235{236 wr_waitp->ret = 0;237 init_completion(&wr_waitp->completion);238}239 240static inline void _c4iw_wake_up(struct c4iw_wr_wait *wr_waitp, int ret,241 bool deref)242{243 wr_waitp->ret = ret;244 complete(&wr_waitp->completion);245 if (deref)246 c4iw_put_wr_wait(wr_waitp);247}248 249static inline void c4iw_wake_up_noref(struct c4iw_wr_wait *wr_waitp, int ret)250{251 _c4iw_wake_up(wr_waitp, ret, false);252}253 254static inline void c4iw_wake_up_deref(struct c4iw_wr_wait *wr_waitp, int ret)255{256 _c4iw_wake_up(wr_waitp, ret, true);257}258 259static inline int c4iw_wait_for_reply(struct c4iw_rdev *rdev,260 struct c4iw_wr_wait *wr_waitp,261 u32 hwtid, u32 qpid,262 const char *func)263{264 int ret;265 266 if (c4iw_fatal_error(rdev)) {267 wr_waitp->ret = -EIO;268 goto out;269 }270 271 ret = wait_for_completion_timeout(&wr_waitp->completion, C4IW_WR_TO);272 if (!ret) {273 pr_err("%s - Device %s not responding (disabling device) - tid %u qpid %u\n",274 func, pci_name(rdev->lldi.pdev), hwtid, qpid);275 rdev->flags |= T4_FATAL_ERROR;276 wr_waitp->ret = -EIO;277 goto out;278 }279 if (wr_waitp->ret)280 pr_debug("%s: FW reply %d tid %u qpid %u\n",281 pci_name(rdev->lldi.pdev), wr_waitp->ret, hwtid, qpid);282out:283 return wr_waitp->ret;284}285 286int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb);287 288static inline int c4iw_ref_send_wait(struct c4iw_rdev *rdev,289 struct sk_buff *skb,290 struct c4iw_wr_wait *wr_waitp,291 u32 hwtid, u32 qpid,292 const char *func)293{294 int ret;295 296 pr_debug("%s wr_wait %p hwtid %u qpid %u\n", func, wr_waitp, hwtid,297 qpid);298 c4iw_get_wr_wait(wr_waitp);299 ret = c4iw_ofld_send(rdev, skb);300 if (ret) {301 c4iw_put_wr_wait(wr_waitp);302 return ret;303 }304 return c4iw_wait_for_reply(rdev, wr_waitp, hwtid, qpid, func);305}306 307enum db_state {308 NORMAL = 0,309 FLOW_CONTROL = 1,310 RECOVERY = 2,311 STOPPED = 3312};313 314struct c4iw_dev {315 struct ib_device ibdev;316 struct c4iw_rdev rdev;317 struct xarray cqs;318 struct xarray qps;319 struct xarray mrs;320 struct mutex db_mutex;321 struct dentry *debugfs_root;322 enum db_state db_state;323 struct xarray hwtids;324 struct xarray atids;325 struct xarray stids;326 struct list_head db_fc_list;327 u32 avail_ird;328 wait_queue_head_t wait;329};330 331struct uld_ctx {332 struct list_head entry;333 struct cxgb4_lld_info lldi;334 struct c4iw_dev *dev;335 struct work_struct reg_work;336};337 338static inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev)339{340 return container_of(ibdev, struct c4iw_dev, ibdev);341}342 343static inline struct c4iw_cq *get_chp(struct c4iw_dev *rhp, u32 cqid)344{345 return xa_load(&rhp->cqs, cqid);346}347 348static inline struct c4iw_qp *get_qhp(struct c4iw_dev *rhp, u32 qpid)349{350 return xa_load(&rhp->qps, qpid);351}352 353extern uint c4iw_max_read_depth;354 355static inline int cur_max_read_depth(struct c4iw_dev *dev)356{357 return min(dev->rdev.lldi.max_ordird_qp, c4iw_max_read_depth);358}359 360struct c4iw_pd {361 struct ib_pd ibpd;362 u32 pdid;363 struct c4iw_dev *rhp;364};365 366static inline struct c4iw_pd *to_c4iw_pd(struct ib_pd *ibpd)367{368 return container_of(ibpd, struct c4iw_pd, ibpd);369}370 371struct tpt_attributes {372 u64 len;373 u64 va_fbo;374 enum fw_ri_mem_perms perms;375 u32 stag;376 u32 pdid;377 u32 qpid;378 u32 pbl_addr;379 u32 pbl_size;380 u32 state:1;381 u32 type:2;382 u32 rsvd:1;383 u32 remote_invaliate_disable:1;384 u32 zbva:1;385 u32 mw_bind_enable:1;386 u32 page_size:5;387};388 389struct c4iw_mr {390 struct ib_mr ibmr;391 struct ib_umem *umem;392 struct c4iw_dev *rhp;393 struct sk_buff *dereg_skb;394 u64 kva;395 struct tpt_attributes attr;396 u64 *mpl;397 dma_addr_t mpl_addr;398 u32 max_mpl_len;399 u32 mpl_len;400 struct c4iw_wr_wait *wr_waitp;401};402 403static inline struct c4iw_mr *to_c4iw_mr(struct ib_mr *ibmr)404{405 return container_of(ibmr, struct c4iw_mr, ibmr);406}407 408struct c4iw_mw {409 struct ib_mw ibmw;410 struct c4iw_dev *rhp;411 struct sk_buff *dereg_skb;412 u64 kva;413 struct tpt_attributes attr;414 struct c4iw_wr_wait *wr_waitp;415};416 417static inline struct c4iw_mw *to_c4iw_mw(struct ib_mw *ibmw)418{419 return container_of(ibmw, struct c4iw_mw, ibmw);420}421 422struct c4iw_cq {423 struct ib_cq ibcq;424 struct c4iw_dev *rhp;425 struct sk_buff *destroy_skb;426 struct t4_cq cq;427 spinlock_t lock;428 spinlock_t comp_handler_lock;429 refcount_t refcnt;430 struct completion cq_rel_comp;431 struct c4iw_wr_wait *wr_waitp;432};433 434static inline struct c4iw_cq *to_c4iw_cq(struct ib_cq *ibcq)435{436 return container_of(ibcq, struct c4iw_cq, ibcq);437}438 439struct c4iw_mpa_attributes {440 u8 initiator;441 u8 recv_marker_enabled;442 u8 xmit_marker_enabled;443 u8 crc_enabled;444 u8 enhanced_rdma_conn;445 u8 version;446 u8 p2p_type;447};448 449struct c4iw_qp_attributes {450 u32 scq;451 u32 rcq;452 u32 sq_num_entries;453 u32 rq_num_entries;454 u32 sq_max_sges;455 u32 sq_max_sges_rdma_write;456 u32 rq_max_sges;457 u32 state;458 u8 enable_rdma_read;459 u8 enable_rdma_write;460 u8 enable_bind;461 u8 enable_mmid0_fastreg;462 u32 max_ord;463 u32 max_ird;464 u32 pd;465 u32 next_state;466 char terminate_buffer[52];467 u32 terminate_msg_len;468 u8 is_terminate_local;469 struct c4iw_mpa_attributes mpa_attr;470 struct c4iw_ep *llp_stream_handle;471 u8 layer_etype;472 u8 ecode;473 u16 sq_db_inc;474 u16 rq_db_inc;475 u8 send_term;476};477 478struct c4iw_qp {479 struct ib_qp ibqp;480 struct list_head db_fc_entry;481 struct c4iw_dev *rhp;482 struct c4iw_ep *ep;483 struct c4iw_qp_attributes attr;484 struct t4_wq wq;485 spinlock_t lock;486 struct mutex mutex;487 wait_queue_head_t wait;488 int sq_sig_all;489 struct c4iw_srq *srq;490 struct c4iw_ucontext *ucontext;491 struct c4iw_wr_wait *wr_waitp;492 struct completion qp_rel_comp;493 refcount_t qp_refcnt;494};495 496static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp)497{498 return container_of(ibqp, struct c4iw_qp, ibqp);499}500 501struct c4iw_srq {502 struct ib_srq ibsrq;503 struct list_head db_fc_entry;504 struct c4iw_dev *rhp;505 struct t4_srq wq;506 struct sk_buff *destroy_skb;507 u32 srq_limit;508 u32 pdid;509 int idx;510 u32 flags;511 spinlock_t lock; /* protects srq */512 struct c4iw_wr_wait *wr_waitp;513 bool armed;514};515 516static inline struct c4iw_srq *to_c4iw_srq(struct ib_srq *ibsrq)517{518 return container_of(ibsrq, struct c4iw_srq, ibsrq);519}520 521struct c4iw_ucontext {522 struct ib_ucontext ibucontext;523 struct c4iw_dev_ucontext uctx;524 u32 key;525 spinlock_t mmap_lock;526 struct list_head mmaps;527 bool is_32b_cqe;528};529 530static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c)531{532 return container_of(c, struct c4iw_ucontext, ibucontext);533}534 535enum {536 CXGB4_MMAP_BAR,537 CXGB4_MMAP_BAR_WC,538 CXGB4_MMAP_CONTIG,539 CXGB4_MMAP_NON_CONTIG,540};541 542struct c4iw_mm_entry {543 struct list_head entry;544 u64 addr;545 u32 key;546 void *vaddr;547 dma_addr_t dma_addr;548 unsigned len;549 u8 mmap_flag;550};551 552static inline struct c4iw_mm_entry *remove_mmap(struct c4iw_ucontext *ucontext,553 u32 key, unsigned len)554{555 struct list_head *pos, *nxt;556 struct c4iw_mm_entry *mm;557 558 spin_lock(&ucontext->mmap_lock);559 list_for_each_safe(pos, nxt, &ucontext->mmaps) {560 561 mm = list_entry(pos, struct c4iw_mm_entry, entry);562 if (mm->key == key && mm->len == len) {563 list_del_init(&mm->entry);564 spin_unlock(&ucontext->mmap_lock);565 pr_debug("key 0x%x addr 0x%llx len %d\n", key,566 (unsigned long long)mm->addr, mm->len);567 return mm;568 }569 }570 spin_unlock(&ucontext->mmap_lock);571 return NULL;572}573 574static inline void insert_flag_to_mmap(struct c4iw_rdev *rdev,575 struct c4iw_mm_entry *mm, u64 addr)576{577 if (addr >= pci_resource_start(rdev->lldi.pdev, 0) &&578 (addr < (pci_resource_start(rdev->lldi.pdev, 0) +579 pci_resource_len(rdev->lldi.pdev, 0))))580 mm->mmap_flag = CXGB4_MMAP_BAR;581 else if (addr >= pci_resource_start(rdev->lldi.pdev, 2) &&582 (addr < (pci_resource_start(rdev->lldi.pdev, 2) +583 pci_resource_len(rdev->lldi.pdev, 2)))) {584 if (addr >= rdev->oc_mw_pa) {585 mm->mmap_flag = CXGB4_MMAP_BAR_WC;586 } else {587 if (is_t4(rdev->lldi.adapter_type))588 mm->mmap_flag = CXGB4_MMAP_BAR;589 else590 mm->mmap_flag = CXGB4_MMAP_BAR_WC;591 }592 } else {593 if (addr)594 mm->mmap_flag = CXGB4_MMAP_CONTIG;595 else596 mm->mmap_flag = CXGB4_MMAP_NON_CONTIG;597 }598}599 600static inline void insert_mmap(struct c4iw_ucontext *ucontext,601 struct c4iw_mm_entry *mm)602{603 spin_lock(&ucontext->mmap_lock);604 pr_debug("key 0x%x addr 0x%llx len %d\n",605 mm->key, (unsigned long long)mm->addr, mm->len);606 list_add_tail(&mm->entry, &ucontext->mmaps);607 spin_unlock(&ucontext->mmap_lock);608}609 610enum c4iw_qp_attr_mask {611 C4IW_QP_ATTR_NEXT_STATE = 1 << 0,612 C4IW_QP_ATTR_SQ_DB = 1<<1,613 C4IW_QP_ATTR_RQ_DB = 1<<2,614 C4IW_QP_ATTR_ENABLE_RDMA_READ = 1 << 7,615 C4IW_QP_ATTR_ENABLE_RDMA_WRITE = 1 << 8,616 C4IW_QP_ATTR_ENABLE_RDMA_BIND = 1 << 9,617 C4IW_QP_ATTR_MAX_ORD = 1 << 11,618 C4IW_QP_ATTR_MAX_IRD = 1 << 12,619 C4IW_QP_ATTR_LLP_STREAM_HANDLE = 1 << 22,620 C4IW_QP_ATTR_STREAM_MSG_BUFFER = 1 << 23,621 C4IW_QP_ATTR_MPA_ATTR = 1 << 24,622 C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE = 1 << 25,623 C4IW_QP_ATTR_VALID_MODIFY = (C4IW_QP_ATTR_ENABLE_RDMA_READ |624 C4IW_QP_ATTR_ENABLE_RDMA_WRITE |625 C4IW_QP_ATTR_MAX_ORD |626 C4IW_QP_ATTR_MAX_IRD |627 C4IW_QP_ATTR_LLP_STREAM_HANDLE |628 C4IW_QP_ATTR_STREAM_MSG_BUFFER |629 C4IW_QP_ATTR_MPA_ATTR |630 C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE)631};632 633int c4iw_modify_qp(struct c4iw_dev *rhp,634 struct c4iw_qp *qhp,635 enum c4iw_qp_attr_mask mask,636 struct c4iw_qp_attributes *attrs,637 int internal);638 639enum c4iw_qp_state {640 C4IW_QP_STATE_IDLE,641 C4IW_QP_STATE_RTS,642 C4IW_QP_STATE_ERROR,643 C4IW_QP_STATE_TERMINATE,644 C4IW_QP_STATE_CLOSING,645 C4IW_QP_STATE_TOT646};647 648static inline int c4iw_convert_state(enum ib_qp_state ib_state)649{650 switch (ib_state) {651 case IB_QPS_RESET:652 case IB_QPS_INIT:653 return C4IW_QP_STATE_IDLE;654 case IB_QPS_RTS:655 return C4IW_QP_STATE_RTS;656 case IB_QPS_SQD:657 return C4IW_QP_STATE_CLOSING;658 case IB_QPS_SQE:659 return C4IW_QP_STATE_TERMINATE;660 case IB_QPS_ERR:661 return C4IW_QP_STATE_ERROR;662 default:663 return -1;664 }665}666 667static inline int to_ib_qp_state(int c4iw_qp_state)668{669 switch (c4iw_qp_state) {670 case C4IW_QP_STATE_IDLE:671 return IB_QPS_INIT;672 case C4IW_QP_STATE_RTS:673 return IB_QPS_RTS;674 case C4IW_QP_STATE_CLOSING:675 return IB_QPS_SQD;676 case C4IW_QP_STATE_TERMINATE:677 return IB_QPS_SQE;678 case C4IW_QP_STATE_ERROR:679 return IB_QPS_ERR;680 }681 return IB_QPS_ERR;682}683 684static inline u32 c4iw_ib_to_tpt_access(int a)685{686 return (a & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) |687 (a & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0) |688 (a & IB_ACCESS_LOCAL_WRITE ? FW_RI_MEM_ACCESS_LOCAL_WRITE : 0) |689 FW_RI_MEM_ACCESS_LOCAL_READ;690}691 692enum c4iw_mmid_state {693 C4IW_STAG_STATE_VALID,694 C4IW_STAG_STATE_INVALID695};696 697#define C4IW_NODE_DESC "cxgb4 Chelsio Communications"698 699#define MPA_KEY_REQ "MPA ID Req Frame"700#define MPA_KEY_REP "MPA ID Rep Frame"701 702#define MPA_MAX_PRIVATE_DATA 256703#define MPA_ENHANCED_RDMA_CONN 0x10704#define MPA_REJECT 0x20705#define MPA_CRC 0x40706#define MPA_MARKERS 0x80707#define MPA_FLAGS_MASK 0xE0708 709#define MPA_V2_PEER2PEER_MODEL 0x8000710#define MPA_V2_ZERO_LEN_FPDU_RTR 0x4000711#define MPA_V2_RDMA_WRITE_RTR 0x8000712#define MPA_V2_RDMA_READ_RTR 0x4000713#define MPA_V2_IRD_ORD_MASK 0x3FFF714 715#define c4iw_put_ep(ep) { \716 pr_debug("put_ep ep %p refcnt %d\n", \717 ep, kref_read(&((ep)->kref))); \718 WARN_ON(kref_read(&((ep)->kref)) < 1); \719 kref_put(&((ep)->kref), _c4iw_free_ep); \720}721 722#define c4iw_get_ep(ep) { \723 pr_debug("get_ep ep %p, refcnt %d\n", \724 ep, kref_read(&((ep)->kref))); \725 kref_get(&((ep)->kref)); \726}727void _c4iw_free_ep(struct kref *kref);728 729struct mpa_message {730 u8 key[16];731 u8 flags;732 u8 revision;733 __be16 private_data_size;734 u8 private_data[];735};736 737struct mpa_v2_conn_params {738 __be16 ird;739 __be16 ord;740};741 742struct terminate_message {743 u8 layer_etype;744 u8 ecode;745 __be16 hdrct_rsvd;746 u8 len_hdrs[];747};748 749#define TERM_MAX_LENGTH (sizeof(struct terminate_message) + 2 + 18 + 28)750 751enum c4iw_layers_types {752 LAYER_RDMAP = 0x00,753 LAYER_DDP = 0x10,754 LAYER_MPA = 0x20,755 RDMAP_LOCAL_CATA = 0x00,756 RDMAP_REMOTE_PROT = 0x01,757 RDMAP_REMOTE_OP = 0x02,758 DDP_LOCAL_CATA = 0x00,759 DDP_TAGGED_ERR = 0x01,760 DDP_UNTAGGED_ERR = 0x02,761 DDP_LLP = 0x03762};763 764enum c4iw_rdma_ecodes {765 RDMAP_INV_STAG = 0x00,766 RDMAP_BASE_BOUNDS = 0x01,767 RDMAP_ACC_VIOL = 0x02,768 RDMAP_STAG_NOT_ASSOC = 0x03,769 RDMAP_TO_WRAP = 0x04,770 RDMAP_INV_VERS = 0x05,771 RDMAP_INV_OPCODE = 0x06,772 RDMAP_STREAM_CATA = 0x07,773 RDMAP_GLOBAL_CATA = 0x08,774 RDMAP_CANT_INV_STAG = 0x09,775 RDMAP_UNSPECIFIED = 0xff776};777 778enum c4iw_ddp_ecodes {779 DDPT_INV_STAG = 0x00,780 DDPT_BASE_BOUNDS = 0x01,781 DDPT_STAG_NOT_ASSOC = 0x02,782 DDPT_TO_WRAP = 0x03,783 DDPT_INV_VERS = 0x04,784 DDPU_INV_QN = 0x01,785 DDPU_INV_MSN_NOBUF = 0x02,786 DDPU_INV_MSN_RANGE = 0x03,787 DDPU_INV_MO = 0x04,788 DDPU_MSG_TOOBIG = 0x05,789 DDPU_INV_VERS = 0x06790};791 792enum c4iw_mpa_ecodes {793 MPA_CRC_ERR = 0x02,794 MPA_MARKER_ERR = 0x03,795 MPA_LOCAL_CATA = 0x05,796 MPA_INSUFF_IRD = 0x06,797 MPA_NOMATCH_RTR = 0x07,798};799 800enum c4iw_ep_state {801 IDLE = 0,802 LISTEN,803 CONNECTING,804 MPA_REQ_WAIT,805 MPA_REQ_SENT,806 MPA_REQ_RCVD,807 MPA_REP_SENT,808 FPDU_MODE,809 ABORTING,810 CLOSING,811 MORIBUND,812 DEAD,813};814 815enum c4iw_ep_flags {816 PEER_ABORT_IN_PROGRESS = 0,817 ABORT_REQ_IN_PROGRESS = 1,818 RELEASE_RESOURCES = 2,819 CLOSE_SENT = 3,820 TIMEOUT = 4,821 QP_REFERENCED = 5,822 STOP_MPA_TIMER = 7,823};824 825enum c4iw_ep_history {826 ACT_OPEN_REQ = 0,827 ACT_OFLD_CONN = 1,828 ACT_OPEN_RPL = 2,829 ACT_ESTAB = 3,830 PASS_ACCEPT_REQ = 4,831 PASS_ESTAB = 5,832 ABORT_UPCALL = 6,833 ESTAB_UPCALL = 7,834 CLOSE_UPCALL = 8,835 ULP_ACCEPT = 9,836 ULP_REJECT = 10,837 TIMEDOUT = 11,838 PEER_ABORT = 12,839 PEER_CLOSE = 13,840 CONNREQ_UPCALL = 14,841 ABORT_CONN = 15,842 DISCONN_UPCALL = 16,843 EP_DISC_CLOSE = 17,844 EP_DISC_ABORT = 18,845 CONN_RPL_UPCALL = 19,846 ACT_RETRY_NOMEM = 20,847 ACT_RETRY_INUSE = 21,848 CLOSE_CON_RPL = 22,849 EP_DISC_FAIL = 24,850 QP_REFED = 25,851 QP_DEREFED = 26,852 CM_ID_REFED = 27,853 CM_ID_DEREFED = 28,854};855 856enum conn_pre_alloc_buffers {857 CN_ABORT_REQ_BUF,858 CN_ABORT_RPL_BUF,859 CN_CLOSE_CON_REQ_BUF,860 CN_DESTROY_BUF,861 CN_FLOWC_BUF,862 CN_MAX_CON_BUF863};864 865enum {866 FLOWC_LEN = offsetof(struct fw_flowc_wr, mnemval[FW_FLOWC_MNEM_MAX])867};868 869union cpl_wr_size {870 struct cpl_abort_req abrt_req;871 struct cpl_abort_rpl abrt_rpl;872 struct fw_ri_wr ri_req;873 struct cpl_close_con_req close_req;874 char flowc_buf[FLOWC_LEN];875};876 877struct c4iw_ep_common {878 struct iw_cm_id *cm_id;879 struct c4iw_qp *qp;880 struct c4iw_dev *dev;881 struct sk_buff_head ep_skb_list;882 enum c4iw_ep_state state;883 struct kref kref;884 struct mutex mutex;885 struct sockaddr_storage local_addr;886 struct sockaddr_storage remote_addr;887 struct c4iw_wr_wait *wr_waitp;888 unsigned long flags;889 unsigned long history;890};891 892struct c4iw_listen_ep {893 struct c4iw_ep_common com;894 unsigned int stid;895 int backlog;896};897 898struct c4iw_ep_stats {899 unsigned connect_neg_adv;900 unsigned abort_neg_adv;901};902 903struct c4iw_ep {904 struct c4iw_ep_common com;905 struct c4iw_ep *parent_ep;906 struct timer_list timer;907 struct list_head entry;908 unsigned int atid;909 u32 hwtid;910 u32 snd_seq;911 u32 rcv_seq;912 struct l2t_entry *l2t;913 struct dst_entry *dst;914 struct sk_buff *mpa_skb;915 struct c4iw_mpa_attributes mpa_attr;916 u8 mpa_pkt[sizeof(struct mpa_message) + MPA_MAX_PRIVATE_DATA];917 unsigned int mpa_pkt_len;918 u32 ird;919 u32 ord;920 u32 smac_idx;921 u32 tx_chan;922 u32 mtu;923 u16 mss;924 u16 emss;925 u16 plen;926 u16 rss_qid;927 u16 txq_idx;928 u16 ctrlq_idx;929 u8 tos;930 u8 retry_with_mpa_v1;931 u8 tried_with_mpa_v1;932 unsigned int retry_count;933 int snd_win;934 int rcv_win;935 u32 snd_wscale;936 struct c4iw_ep_stats stats;937 u32 srqe_idx;938 u32 rx_pdu_out_cnt;939 struct sk_buff *peer_abort_skb;940};941 942static inline struct c4iw_ep *to_ep(struct iw_cm_id *cm_id)943{944 return cm_id->provider_data;945}946 947static inline struct c4iw_listen_ep *to_listen_ep(struct iw_cm_id *cm_id)948{949 return cm_id->provider_data;950}951 952static inline int ocqp_supported(const struct cxgb4_lld_info *infop)953{954#if defined(__i386__) || defined(__x86_64__) || defined(CONFIG_PPC64)955 return infop->vr->ocq.size > 0;956#else957 return 0;958#endif959}960 961u32 c4iw_id_alloc(struct c4iw_id_table *alloc);962void c4iw_id_free(struct c4iw_id_table *alloc, u32 obj);963int c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num,964 u32 reserved, u32 flags);965void c4iw_id_table_free(struct c4iw_id_table *alloc);966 967typedef int (*c4iw_handler_func)(struct c4iw_dev *dev, struct sk_buff *skb);968 969void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qpid,970 struct c4iw_dev_ucontext *uctx);971u32 c4iw_get_resource(struct c4iw_id_table *id_table);972void c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry);973int c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt,974 u32 nr_pdid, u32 nr_srqt);975int c4iw_pblpool_create(struct c4iw_rdev *rdev);976int c4iw_rqtpool_create(struct c4iw_rdev *rdev);977int c4iw_ocqp_pool_create(struct c4iw_rdev *rdev);978void c4iw_pblpool_destroy(struct c4iw_rdev *rdev);979void c4iw_rqtpool_destroy(struct c4iw_rdev *rdev);980void c4iw_ocqp_pool_destroy(struct c4iw_rdev *rdev);981void c4iw_destroy_resource(struct c4iw_resource *rscp);982void c4iw_register_device(struct work_struct *work);983void c4iw_unregister_device(struct c4iw_dev *dev);984int __init c4iw_cm_init(void);985void c4iw_cm_term(void);986void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev,987 struct c4iw_dev_ucontext *uctx);988void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev,989 struct c4iw_dev_ucontext *uctx);990int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);991int c4iw_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,992 const struct ib_send_wr **bad_wr);993int c4iw_post_receive(struct ib_qp *ibqp, const struct ib_recv_wr *wr,994 const struct ib_recv_wr **bad_wr);995int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);996int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog);997int c4iw_destroy_listen(struct iw_cm_id *cm_id);998int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);999int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);1000void c4iw_qp_add_ref(struct ib_qp *qp);1001void c4iw_qp_rem_ref(struct ib_qp *qp);1002struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,1003 u32 max_num_sg);1004int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,1005 unsigned int *sg_offset);1006void c4iw_dealloc(struct uld_ctx *ctx);1007struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start,1008 u64 length, u64 virt, int acc,1009 struct ib_udata *udata);1010struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);1011int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);1012int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);1013void c4iw_cq_rem_ref(struct c4iw_cq *chp);1014int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,1015 struct uverbs_attr_bundle *attrs);1016int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);1017int c4iw_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *attr,1018 enum ib_srq_attr_mask srq_attr_mask,1019 struct ib_udata *udata);1020int c4iw_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata);1021int c4iw_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *attrs,1022 struct ib_udata *udata);1023int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata);1024int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs,1025 struct ib_udata *udata);1026int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,1027 int attr_mask, struct ib_udata *udata);1028int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,1029 int attr_mask, struct ib_qp_init_attr *init_attr);1030struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn);1031u32 c4iw_rqtpool_alloc(struct c4iw_rdev *rdev, int size);1032void c4iw_rqtpool_free(struct c4iw_rdev *rdev, u32 addr, int size);1033u32 c4iw_pblpool_alloc(struct c4iw_rdev *rdev, int size);1034void c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size);1035u32 c4iw_ocqp_pool_alloc(struct c4iw_rdev *rdev, int size);1036void c4iw_ocqp_pool_free(struct c4iw_rdev *rdev, u32 addr, int size);1037void c4iw_flush_hw_cq(struct c4iw_cq *chp, struct c4iw_qp *flush_qhp);1038void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count);1039int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp);1040int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count);1041int c4iw_flush_sq(struct c4iw_qp *qhp);1042int c4iw_ev_handler(struct c4iw_dev *rnicp, u32 qid);1043u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);1044void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid,1045 struct c4iw_dev_ucontext *uctx);1046u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);1047void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid,1048 struct c4iw_dev_ucontext *uctx);1049void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe);1050 1051extern struct cxgb4_client t4c_client;1052extern c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS];1053void __iomem *c4iw_bar2_addrs(struct c4iw_rdev *rdev, unsigned int qid,1054 enum cxgb4_bar2_qtype qtype,1055 unsigned int *pbar2_qid, u64 *pbar2_pa);1056int c4iw_alloc_srq_idx(struct c4iw_rdev *rdev);1057void c4iw_free_srq_idx(struct c4iw_rdev *rdev, int idx);1058extern void c4iw_log_wr_stats(struct t4_wq *wq, struct t4_cqe *cqe);1059extern int c4iw_wr_log;1060extern int db_fc_threshold;1061extern int db_coalescing_threshold;1062extern int use_dsgl;1063void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey);1064void c4iw_dispatch_srq_limit_reached_event(struct c4iw_srq *srq);1065void c4iw_copy_wr_to_srq(struct t4_srq *srq, union t4_recv_wr *wqe, u8 len16);1066void c4iw_flush_srqidx(struct c4iw_qp *qhp, u32 srqidx);1067int c4iw_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,1068 const struct ib_recv_wr **bad_wr);1069struct c4iw_wr_wait *c4iw_alloc_wr_wait(gfp_t gfp);1070 1071int c4iw_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr);1072int c4iw_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq);1073int c4iw_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp);1074int c4iw_fill_res_cm_id_entry(struct sk_buff *msg, struct rdma_cm_id *cm_id);1075 1076#endif1077