204 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause2 3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */4/* Kai Shen <kaishen@linux.alibaba.com> */5/* Copyright (c) 2020-2022, Alibaba Group. */6 7#include "erdma_verbs.h"8 9static void *get_next_valid_cqe(struct erdma_cq *cq)10{11 __be32 *cqe = get_queue_entry(cq->kern_cq.qbuf, cq->kern_cq.ci,12 cq->depth, CQE_SHIFT);13 u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,14 be32_to_cpu(READ_ONCE(*cqe)));15 16 return owner ^ !!(cq->kern_cq.ci & cq->depth) ? cqe : NULL;17}18 19static void notify_cq(struct erdma_cq *cq, u8 solcitied)20{21 u64 db_data =22 FIELD_PREP(ERDMA_CQDB_IDX_MASK, (cq->kern_cq.notify_cnt)) |23 FIELD_PREP(ERDMA_CQDB_CQN_MASK, cq->cqn) |24 FIELD_PREP(ERDMA_CQDB_ARM_MASK, 1) |25 FIELD_PREP(ERDMA_CQDB_SOL_MASK, solcitied) |26 FIELD_PREP(ERDMA_CQDB_CMDSN_MASK, cq->kern_cq.cmdsn) |27 FIELD_PREP(ERDMA_CQDB_CI_MASK, cq->kern_cq.ci);28 29 *cq->kern_cq.dbrec = db_data;30 writeq(db_data, cq->kern_cq.db);31}32 33int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)34{35 struct erdma_cq *cq = to_ecq(ibcq);36 unsigned long irq_flags;37 int ret = 0;38 39 spin_lock_irqsave(&cq->kern_cq.lock, irq_flags);40 41 notify_cq(cq, (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED);42 43 if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && get_next_valid_cqe(cq))44 ret = 1;45 46 cq->kern_cq.notify_cnt++;47 48 spin_unlock_irqrestore(&cq->kern_cq.lock, irq_flags);49 50 return ret;51}52 53static const enum ib_wc_opcode wc_mapping_table[ERDMA_NUM_OPCODES] = {54 [ERDMA_OP_WRITE] = IB_WC_RDMA_WRITE,55 [ERDMA_OP_READ] = IB_WC_RDMA_READ,56 [ERDMA_OP_SEND] = IB_WC_SEND,57 [ERDMA_OP_SEND_WITH_IMM] = IB_WC_SEND,58 [ERDMA_OP_RECEIVE] = IB_WC_RECV,59 [ERDMA_OP_RECV_IMM] = IB_WC_RECV_RDMA_WITH_IMM,60 [ERDMA_OP_RECV_INV] = IB_WC_RECV,61 [ERDMA_OP_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE,62 [ERDMA_OP_RSP_SEND_IMM] = IB_WC_RECV,63 [ERDMA_OP_SEND_WITH_INV] = IB_WC_SEND,64 [ERDMA_OP_REG_MR] = IB_WC_REG_MR,65 [ERDMA_OP_LOCAL_INV] = IB_WC_LOCAL_INV,66 [ERDMA_OP_READ_WITH_INV] = IB_WC_RDMA_READ,67 [ERDMA_OP_ATOMIC_CAS] = IB_WC_COMP_SWAP,68 [ERDMA_OP_ATOMIC_FAA] = IB_WC_FETCH_ADD,69};70 71static const struct {72 enum erdma_wc_status erdma;73 enum ib_wc_status base;74 enum erdma_vendor_err vendor;75} map_cqe_status[ERDMA_NUM_WC_STATUS] = {76 { ERDMA_WC_SUCCESS, IB_WC_SUCCESS, ERDMA_WC_VENDOR_NO_ERR },77 { ERDMA_WC_GENERAL_ERR, IB_WC_GENERAL_ERR, ERDMA_WC_VENDOR_NO_ERR },78 { ERDMA_WC_RECV_WQE_FORMAT_ERR, IB_WC_GENERAL_ERR,79 ERDMA_WC_VENDOR_INVALID_RQE },80 { ERDMA_WC_RECV_STAG_INVALID_ERR, IB_WC_REM_ACCESS_ERR,81 ERDMA_WC_VENDOR_RQE_INVALID_STAG },82 { ERDMA_WC_RECV_ADDR_VIOLATION_ERR, IB_WC_REM_ACCESS_ERR,83 ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION },84 { ERDMA_WC_RECV_RIGHT_VIOLATION_ERR, IB_WC_REM_ACCESS_ERR,85 ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR },86 { ERDMA_WC_RECV_PDID_ERR, IB_WC_REM_ACCESS_ERR,87 ERDMA_WC_VENDOR_RQE_INVALID_PD },88 { ERDMA_WC_RECV_WARRPING_ERR, IB_WC_REM_ACCESS_ERR,89 ERDMA_WC_VENDOR_RQE_WRAP_ERR },90 { ERDMA_WC_SEND_WQE_FORMAT_ERR, IB_WC_LOC_QP_OP_ERR,91 ERDMA_WC_VENDOR_INVALID_SQE },92 { ERDMA_WC_SEND_WQE_ORD_EXCEED, IB_WC_GENERAL_ERR,93 ERDMA_WC_VENDOR_ZERO_ORD },94 { ERDMA_WC_SEND_STAG_INVALID_ERR, IB_WC_LOC_ACCESS_ERR,95 ERDMA_WC_VENDOR_SQE_INVALID_STAG },96 { ERDMA_WC_SEND_ADDR_VIOLATION_ERR, IB_WC_LOC_ACCESS_ERR,97 ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION },98 { ERDMA_WC_SEND_RIGHT_VIOLATION_ERR, IB_WC_LOC_ACCESS_ERR,99 ERDMA_WC_VENDOR_SQE_ACCESS_ERR },100 { ERDMA_WC_SEND_PDID_ERR, IB_WC_LOC_ACCESS_ERR,101 ERDMA_WC_VENDOR_SQE_INVALID_PD },102 { ERDMA_WC_SEND_WARRPING_ERR, IB_WC_LOC_ACCESS_ERR,103 ERDMA_WC_VENDOR_SQE_WARP_ERR },104 { ERDMA_WC_FLUSH_ERR, IB_WC_WR_FLUSH_ERR, ERDMA_WC_VENDOR_NO_ERR },105 { ERDMA_WC_RETRY_EXC_ERR, IB_WC_RETRY_EXC_ERR, ERDMA_WC_VENDOR_NO_ERR },106};107 108#define ERDMA_POLLCQ_NO_QP 1109 110static int erdma_poll_one_cqe(struct erdma_cq *cq, struct ib_wc *wc)111{112 struct erdma_dev *dev = to_edev(cq->ibcq.device);113 u8 opcode, syndrome, qtype;114 struct erdma_kqp *kern_qp;115 struct erdma_cqe *cqe;116 struct erdma_qp *qp;117 u16 wqe_idx, depth;118 u32 qpn, cqe_hdr;119 u64 *id_table;120 u64 *wqe_hdr;121 122 cqe = get_next_valid_cqe(cq);123 if (!cqe)124 return -EAGAIN;125 126 cq->kern_cq.ci++;127 128 /* cqbuf should be ready when we poll */129 dma_rmb();130 131 qpn = be32_to_cpu(cqe->qpn);132 wqe_idx = be32_to_cpu(cqe->qe_idx);133 cqe_hdr = be32_to_cpu(cqe->hdr);134 135 qp = find_qp_by_qpn(dev, qpn);136 if (!qp)137 return ERDMA_POLLCQ_NO_QP;138 139 kern_qp = &qp->kern_qp;140 141 qtype = FIELD_GET(ERDMA_CQE_HDR_QTYPE_MASK, cqe_hdr);142 syndrome = FIELD_GET(ERDMA_CQE_HDR_SYNDROME_MASK, cqe_hdr);143 opcode = FIELD_GET(ERDMA_CQE_HDR_OPCODE_MASK, cqe_hdr);144 145 if (qtype == ERDMA_CQE_QTYPE_SQ) {146 id_table = kern_qp->swr_tbl;147 depth = qp->attrs.sq_size;148 wqe_hdr = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,149 qp->attrs.sq_size, SQEBB_SHIFT);150 kern_qp->sq_ci =151 FIELD_GET(ERDMA_SQE_HDR_WQEBB_CNT_MASK, *wqe_hdr) +152 wqe_idx + 1;153 } else {154 id_table = kern_qp->rwr_tbl;155 depth = qp->attrs.rq_size;156 }157 wc->wr_id = id_table[wqe_idx & (depth - 1)];158 wc->byte_len = be32_to_cpu(cqe->size);159 160 wc->wc_flags = 0;161 162 wc->opcode = wc_mapping_table[opcode];163 if (opcode == ERDMA_OP_RECV_IMM || opcode == ERDMA_OP_RSP_SEND_IMM) {164 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(cqe->imm_data));165 wc->wc_flags |= IB_WC_WITH_IMM;166 } else if (opcode == ERDMA_OP_RECV_INV) {167 wc->ex.invalidate_rkey = be32_to_cpu(cqe->inv_rkey);168 wc->wc_flags |= IB_WC_WITH_INVALIDATE;169 }170 171 if (syndrome >= ERDMA_NUM_WC_STATUS)172 syndrome = ERDMA_WC_GENERAL_ERR;173 174 wc->status = map_cqe_status[syndrome].base;175 wc->vendor_err = map_cqe_status[syndrome].vendor;176 wc->qp = &qp->ibqp;177 178 return 0;179}180 181int erdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)182{183 struct erdma_cq *cq = to_ecq(ibcq);184 unsigned long flags;185 int npolled, ret;186 187 spin_lock_irqsave(&cq->kern_cq.lock, flags);188 189 for (npolled = 0; npolled < num_entries;) {190 ret = erdma_poll_one_cqe(cq, wc + npolled);191 192 if (ret == -EAGAIN) /* no received new CQEs. */193 break;194 else if (ret) /* ignore invalid CQEs. */195 continue;196 197 npolled++;198 }199 200 spin_unlock_irqrestore(&cq->kern_cq.lock, flags);201 202 return npolled;203}204