846 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB2/*3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.5 */6 7#include <linux/skbuff.h>8 9#include "rxe.h"10#include "rxe_loc.h"11#include "rxe_queue.h"12#include "rxe_task.h"13 14enum comp_state {15 COMPST_GET_ACK,16 COMPST_GET_WQE,17 COMPST_COMP_WQE,18 COMPST_COMP_ACK,19 COMPST_CHECK_PSN,20 COMPST_CHECK_ACK,21 COMPST_READ,22 COMPST_ATOMIC,23 COMPST_WRITE_SEND,24 COMPST_UPDATE_COMP,25 COMPST_ERROR_RETRY,26 COMPST_RNR_RETRY,27 COMPST_ERROR,28 COMPST_EXIT, /* We have an issue, and we want to rerun the completer */29 COMPST_DONE, /* The completer finished successflly */30};31 32static char *comp_state_name[] = {33 [COMPST_GET_ACK] = "GET ACK",34 [COMPST_GET_WQE] = "GET WQE",35 [COMPST_COMP_WQE] = "COMP WQE",36 [COMPST_COMP_ACK] = "COMP ACK",37 [COMPST_CHECK_PSN] = "CHECK PSN",38 [COMPST_CHECK_ACK] = "CHECK ACK",39 [COMPST_READ] = "READ",40 [COMPST_ATOMIC] = "ATOMIC",41 [COMPST_WRITE_SEND] = "WRITE/SEND",42 [COMPST_UPDATE_COMP] = "UPDATE COMP",43 [COMPST_ERROR_RETRY] = "ERROR RETRY",44 [COMPST_RNR_RETRY] = "RNR RETRY",45 [COMPST_ERROR] = "ERROR",46 [COMPST_EXIT] = "EXIT",47 [COMPST_DONE] = "DONE",48};49 50static unsigned long rnrnak_usec[32] = {51 [IB_RNR_TIMER_655_36] = 655360,52 [IB_RNR_TIMER_000_01] = 10,53 [IB_RNR_TIMER_000_02] = 20,54 [IB_RNR_TIMER_000_03] = 30,55 [IB_RNR_TIMER_000_04] = 40,56 [IB_RNR_TIMER_000_06] = 60,57 [IB_RNR_TIMER_000_08] = 80,58 [IB_RNR_TIMER_000_12] = 120,59 [IB_RNR_TIMER_000_16] = 160,60 [IB_RNR_TIMER_000_24] = 240,61 [IB_RNR_TIMER_000_32] = 320,62 [IB_RNR_TIMER_000_48] = 480,63 [IB_RNR_TIMER_000_64] = 640,64 [IB_RNR_TIMER_000_96] = 960,65 [IB_RNR_TIMER_001_28] = 1280,66 [IB_RNR_TIMER_001_92] = 1920,67 [IB_RNR_TIMER_002_56] = 2560,68 [IB_RNR_TIMER_003_84] = 3840,69 [IB_RNR_TIMER_005_12] = 5120,70 [IB_RNR_TIMER_007_68] = 7680,71 [IB_RNR_TIMER_010_24] = 10240,72 [IB_RNR_TIMER_015_36] = 15360,73 [IB_RNR_TIMER_020_48] = 20480,74 [IB_RNR_TIMER_030_72] = 30720,75 [IB_RNR_TIMER_040_96] = 40960,76 [IB_RNR_TIMER_061_44] = 61410,77 [IB_RNR_TIMER_081_92] = 81920,78 [IB_RNR_TIMER_122_88] = 122880,79 [IB_RNR_TIMER_163_84] = 163840,80 [IB_RNR_TIMER_245_76] = 245760,81 [IB_RNR_TIMER_327_68] = 327680,82 [IB_RNR_TIMER_491_52] = 491520,83};84 85static inline unsigned long rnrnak_jiffies(u8 timeout)86{87 return max_t(unsigned long,88 usecs_to_jiffies(rnrnak_usec[timeout]), 1);89}90 91static enum ib_wc_opcode wr_to_wc_opcode(enum ib_wr_opcode opcode)92{93 switch (opcode) {94 case IB_WR_RDMA_WRITE: return IB_WC_RDMA_WRITE;95 case IB_WR_RDMA_WRITE_WITH_IMM: return IB_WC_RDMA_WRITE;96 case IB_WR_SEND: return IB_WC_SEND;97 case IB_WR_SEND_WITH_IMM: return IB_WC_SEND;98 case IB_WR_RDMA_READ: return IB_WC_RDMA_READ;99 case IB_WR_ATOMIC_CMP_AND_SWP: return IB_WC_COMP_SWAP;100 case IB_WR_ATOMIC_FETCH_AND_ADD: return IB_WC_FETCH_ADD;101 case IB_WR_LSO: return IB_WC_LSO;102 case IB_WR_SEND_WITH_INV: return IB_WC_SEND;103 case IB_WR_RDMA_READ_WITH_INV: return IB_WC_RDMA_READ;104 case IB_WR_LOCAL_INV: return IB_WC_LOCAL_INV;105 case IB_WR_REG_MR: return IB_WC_REG_MR;106 case IB_WR_BIND_MW: return IB_WC_BIND_MW;107 case IB_WR_ATOMIC_WRITE: return IB_WC_ATOMIC_WRITE;108 case IB_WR_FLUSH: return IB_WC_FLUSH;109 110 default:111 return 0xff;112 }113}114 115void retransmit_timer(struct timer_list *t)116{117 struct rxe_qp *qp = from_timer(qp, t, retrans_timer);118 unsigned long flags;119 120 rxe_dbg_qp(qp, "retransmit timer fired\n");121 122 spin_lock_irqsave(&qp->state_lock, flags);123 if (qp->valid) {124 qp->comp.timeout = 1;125 rxe_sched_task(&qp->send_task);126 }127 spin_unlock_irqrestore(&qp->state_lock, flags);128}129 130void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb)131{132 rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_SENDER_SCHED);133 skb_queue_tail(&qp->resp_pkts, skb);134 rxe_sched_task(&qp->send_task);135}136 137static inline enum comp_state get_wqe(struct rxe_qp *qp,138 struct rxe_pkt_info *pkt,139 struct rxe_send_wqe **wqe_p)140{141 struct rxe_send_wqe *wqe;142 143 /* we come here whether or not we found a response packet to see if144 * there are any posted WQEs145 */146 wqe = queue_head(qp->sq.queue, QUEUE_TYPE_FROM_CLIENT);147 *wqe_p = wqe;148 149 /* no WQE or requester has not started it yet */150 if (!wqe || wqe->state == wqe_state_posted)151 return pkt ? COMPST_DONE : COMPST_EXIT;152 153 /* WQE does not require an ack */154 if (wqe->state == wqe_state_done)155 return COMPST_COMP_WQE;156 157 /* WQE caused an error */158 if (wqe->state == wqe_state_error)159 return COMPST_ERROR;160 161 /* we have a WQE, if we also have an ack check its PSN */162 return pkt ? COMPST_CHECK_PSN : COMPST_EXIT;163}164 165static inline void reset_retry_counters(struct rxe_qp *qp)166{167 qp->comp.retry_cnt = qp->attr.retry_cnt;168 qp->comp.rnr_retry = qp->attr.rnr_retry;169 qp->comp.started_retry = 0;170}171 172static inline enum comp_state check_psn(struct rxe_qp *qp,173 struct rxe_pkt_info *pkt,174 struct rxe_send_wqe *wqe)175{176 s32 diff;177 178 /* check to see if response is past the oldest WQE. if it is, complete179 * send/write or error read/atomic180 */181 diff = psn_compare(pkt->psn, wqe->last_psn);182 if (diff > 0) {183 if (wqe->state == wqe_state_pending) {184 if (wqe->mask & WR_ATOMIC_OR_READ_MASK)185 return COMPST_ERROR_RETRY;186 187 reset_retry_counters(qp);188 return COMPST_COMP_WQE;189 } else {190 return COMPST_DONE;191 }192 }193 194 /* compare response packet to expected response */195 diff = psn_compare(pkt->psn, qp->comp.psn);196 if (diff < 0) {197 /* response is most likely a retried packet if it matches an198 * uncompleted WQE go complete it else ignore it199 */200 if (pkt->psn == wqe->last_psn)201 return COMPST_COMP_ACK;202 else if (pkt->opcode == IB_OPCODE_RC_ACKNOWLEDGE &&203 (qp->comp.opcode == IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST ||204 qp->comp.opcode == IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE))205 return COMPST_CHECK_ACK;206 else207 return COMPST_DONE;208 } else if ((diff > 0) && (wqe->mask & WR_ATOMIC_OR_READ_MASK)) {209 return COMPST_DONE;210 } else {211 return COMPST_CHECK_ACK;212 }213}214 215static inline enum comp_state check_ack(struct rxe_qp *qp,216 struct rxe_pkt_info *pkt,217 struct rxe_send_wqe *wqe)218{219 unsigned int mask = pkt->mask;220 u8 syn;221 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);222 223 /* Check the sequence only */224 switch (qp->comp.opcode) {225 case -1:226 /* Will catch all *_ONLY cases. */227 if (!(mask & RXE_START_MASK))228 return COMPST_ERROR;229 230 break;231 232 case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:233 case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:234 /* Check NAK code to handle a remote error */235 if (pkt->opcode == IB_OPCODE_RC_ACKNOWLEDGE)236 break;237 238 if (pkt->opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE &&239 pkt->opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST) {240 /* read retries of partial data may restart from241 * read response first or response only.242 */243 if ((pkt->psn == wqe->first_psn &&244 pkt->opcode ==245 IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST) ||246 (wqe->first_psn == wqe->last_psn &&247 pkt->opcode ==248 IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY))249 break;250 251 return COMPST_ERROR;252 }253 break;254 default:255 WARN_ON_ONCE(1);256 }257 258 /* Check operation validity. */259 switch (pkt->opcode) {260 case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:261 case IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST:262 case IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY:263 syn = aeth_syn(pkt);264 265 if ((syn & AETH_TYPE_MASK) != AETH_ACK)266 return COMPST_ERROR;267 268 if (wqe->wr.opcode == IB_WR_ATOMIC_WRITE)269 return COMPST_WRITE_SEND;270 271 fallthrough;272 /* (IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE doesn't have an AETH)273 */274 case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:275 if (wqe->wr.opcode != IB_WR_RDMA_READ &&276 wqe->wr.opcode != IB_WR_RDMA_READ_WITH_INV &&277 wqe->wr.opcode != IB_WR_FLUSH) {278 wqe->status = IB_WC_FATAL_ERR;279 return COMPST_ERROR;280 }281 reset_retry_counters(qp);282 return COMPST_READ;283 284 case IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE:285 syn = aeth_syn(pkt);286 287 if ((syn & AETH_TYPE_MASK) != AETH_ACK)288 return COMPST_ERROR;289 290 if (wqe->wr.opcode != IB_WR_ATOMIC_CMP_AND_SWP &&291 wqe->wr.opcode != IB_WR_ATOMIC_FETCH_AND_ADD)292 return COMPST_ERROR;293 reset_retry_counters(qp);294 return COMPST_ATOMIC;295 296 case IB_OPCODE_RC_ACKNOWLEDGE:297 syn = aeth_syn(pkt);298 switch (syn & AETH_TYPE_MASK) {299 case AETH_ACK:300 reset_retry_counters(qp);301 return COMPST_WRITE_SEND;302 303 case AETH_RNR_NAK:304 rxe_counter_inc(rxe, RXE_CNT_RCV_RNR);305 return COMPST_RNR_RETRY;306 307 case AETH_NAK:308 switch (syn) {309 case AETH_NAK_PSN_SEQ_ERROR:310 /* a nak implicitly acks all packets with psns311 * before312 */313 if (psn_compare(pkt->psn, qp->comp.psn) > 0) {314 rxe_counter_inc(rxe,315 RXE_CNT_RCV_SEQ_ERR);316 qp->comp.psn = pkt->psn;317 if (qp->req.wait_psn) {318 qp->req.wait_psn = 0;319 qp->req.again = 1;320 }321 }322 return COMPST_ERROR_RETRY;323 324 case AETH_NAK_INVALID_REQ:325 wqe->status = IB_WC_REM_INV_REQ_ERR;326 return COMPST_ERROR;327 328 case AETH_NAK_REM_ACC_ERR:329 wqe->status = IB_WC_REM_ACCESS_ERR;330 return COMPST_ERROR;331 332 case AETH_NAK_REM_OP_ERR:333 wqe->status = IB_WC_REM_OP_ERR;334 return COMPST_ERROR;335 336 default:337 rxe_dbg_qp(qp, "unexpected nak %x\n", syn);338 wqe->status = IB_WC_REM_OP_ERR;339 return COMPST_ERROR;340 }341 342 default:343 return COMPST_ERROR;344 }345 break;346 347 default:348 rxe_dbg_qp(qp, "unexpected opcode\n");349 }350 351 return COMPST_ERROR;352}353 354static inline enum comp_state do_read(struct rxe_qp *qp,355 struct rxe_pkt_info *pkt,356 struct rxe_send_wqe *wqe)357{358 int ret;359 360 ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE,361 &wqe->dma, payload_addr(pkt),362 payload_size(pkt), RXE_TO_MR_OBJ);363 if (ret) {364 wqe->status = IB_WC_LOC_PROT_ERR;365 return COMPST_ERROR;366 }367 368 if (wqe->dma.resid == 0 && (pkt->mask & RXE_END_MASK))369 return COMPST_COMP_ACK;370 371 return COMPST_UPDATE_COMP;372}373 374static inline enum comp_state do_atomic(struct rxe_qp *qp,375 struct rxe_pkt_info *pkt,376 struct rxe_send_wqe *wqe)377{378 int ret;379 380 u64 atomic_orig = atmack_orig(pkt);381 382 ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE,383 &wqe->dma, &atomic_orig,384 sizeof(u64), RXE_TO_MR_OBJ);385 if (ret) {386 wqe->status = IB_WC_LOC_PROT_ERR;387 return COMPST_ERROR;388 }389 390 return COMPST_COMP_ACK;391}392 393static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe,394 struct rxe_cqe *cqe)395{396 struct ib_wc *wc = &cqe->ibwc;397 struct ib_uverbs_wc *uwc = &cqe->uibwc;398 399 memset(cqe, 0, sizeof(*cqe));400 401 if (!qp->is_user) {402 wc->wr_id = wqe->wr.wr_id;403 wc->status = wqe->status;404 wc->qp = &qp->ibqp;405 } else {406 uwc->wr_id = wqe->wr.wr_id;407 uwc->status = wqe->status;408 uwc->qp_num = qp->ibqp.qp_num;409 }410 411 if (wqe->status == IB_WC_SUCCESS) {412 if (!qp->is_user) {413 wc->opcode = wr_to_wc_opcode(wqe->wr.opcode);414 if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||415 wqe->wr.opcode == IB_WR_SEND_WITH_IMM)416 wc->wc_flags = IB_WC_WITH_IMM;417 wc->byte_len = wqe->dma.length;418 } else {419 uwc->opcode = wr_to_wc_opcode(wqe->wr.opcode);420 if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||421 wqe->wr.opcode == IB_WR_SEND_WITH_IMM)422 uwc->wc_flags = IB_WC_WITH_IMM;423 uwc->byte_len = wqe->dma.length;424 }425 } else {426 if (wqe->status != IB_WC_WR_FLUSH_ERR)427 rxe_err_qp(qp, "non-flush error status = %d\n",428 wqe->status);429 }430}431 432/*433 * IBA Spec. Section 10.7.3.1 SIGNALED COMPLETIONS434 * ---------8<---------8<-------------435 * ...Note that if a completion error occurs, a Work Completion436 * will always be generated, even if the signaling437 * indicator requests an Unsignaled Completion.438 * ---------8<---------8<-------------439 */440static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe)441{442 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);443 struct rxe_cqe cqe;444 bool post;445 446 /* do we need to post a completion */447 post = ((qp->sq_sig_type == IB_SIGNAL_ALL_WR) ||448 (wqe->wr.send_flags & IB_SEND_SIGNALED) ||449 wqe->status != IB_WC_SUCCESS);450 451 if (post)452 make_send_cqe(qp, wqe, &cqe);453 454 queue_advance_consumer(qp->sq.queue, QUEUE_TYPE_FROM_CLIENT);455 456 if (post)457 rxe_cq_post(qp->scq, &cqe, 0);458 459 if (wqe->wr.opcode == IB_WR_SEND ||460 wqe->wr.opcode == IB_WR_SEND_WITH_IMM ||461 wqe->wr.opcode == IB_WR_SEND_WITH_INV)462 rxe_counter_inc(rxe, RXE_CNT_RDMA_SEND);463 464 /*465 * we completed something so let req run again466 * if it is trying to fence467 */468 if (qp->req.wait_fence) {469 qp->req.wait_fence = 0;470 qp->req.again = 1;471 }472}473 474static void comp_check_sq_drain_done(struct rxe_qp *qp)475{476 unsigned long flags;477 478 spin_lock_irqsave(&qp->state_lock, flags);479 if (unlikely(qp_state(qp) == IB_QPS_SQD)) {480 if (qp->attr.sq_draining && qp->comp.psn == qp->req.psn) {481 qp->attr.sq_draining = 0;482 spin_unlock_irqrestore(&qp->state_lock, flags);483 484 if (qp->ibqp.event_handler) {485 struct ib_event ev;486 487 ev.device = qp->ibqp.device;488 ev.element.qp = &qp->ibqp;489 ev.event = IB_EVENT_SQ_DRAINED;490 qp->ibqp.event_handler(&ev,491 qp->ibqp.qp_context);492 }493 return;494 }495 }496 spin_unlock_irqrestore(&qp->state_lock, flags);497}498 499static inline enum comp_state complete_ack(struct rxe_qp *qp,500 struct rxe_pkt_info *pkt,501 struct rxe_send_wqe *wqe)502{503 if (wqe->has_rd_atomic) {504 wqe->has_rd_atomic = 0;505 atomic_inc(&qp->req.rd_atomic);506 if (qp->req.need_rd_atomic) {507 qp->comp.timeout_retry = 0;508 qp->req.need_rd_atomic = 0;509 qp->req.again = 1;510 }511 }512 513 comp_check_sq_drain_done(qp);514 515 do_complete(qp, wqe);516 517 if (psn_compare(pkt->psn, qp->comp.psn) >= 0)518 return COMPST_UPDATE_COMP;519 else520 return COMPST_DONE;521}522 523static inline enum comp_state complete_wqe(struct rxe_qp *qp,524 struct rxe_pkt_info *pkt,525 struct rxe_send_wqe *wqe)526{527 if (pkt && wqe->state == wqe_state_pending) {528 if (psn_compare(wqe->last_psn, qp->comp.psn) >= 0) {529 qp->comp.psn = (wqe->last_psn + 1) & BTH_PSN_MASK;530 qp->comp.opcode = -1;531 }532 533 if (qp->req.wait_psn) {534 qp->req.wait_psn = 0;535 qp->req.again = 1;536 }537 }538 539 do_complete(qp, wqe);540 541 return COMPST_GET_WQE;542}543 544/* drain incoming response packet queue */545static void drain_resp_pkts(struct rxe_qp *qp)546{547 struct sk_buff *skb;548 549 while ((skb = skb_dequeue(&qp->resp_pkts))) {550 rxe_put(qp);551 kfree_skb(skb);552 ib_device_put(qp->ibqp.device);553 }554}555 556/* complete send wqe with flush error */557static int flush_send_wqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe)558{559 struct rxe_cqe cqe = {};560 struct ib_wc *wc = &cqe.ibwc;561 struct ib_uverbs_wc *uwc = &cqe.uibwc;562 int err;563 564 if (qp->is_user) {565 uwc->wr_id = wqe->wr.wr_id;566 uwc->status = IB_WC_WR_FLUSH_ERR;567 uwc->qp_num = qp->ibqp.qp_num;568 } else {569 wc->wr_id = wqe->wr.wr_id;570 wc->status = IB_WC_WR_FLUSH_ERR;571 wc->qp = &qp->ibqp;572 }573 574 err = rxe_cq_post(qp->scq, &cqe, 0);575 if (err)576 rxe_dbg_cq(qp->scq, "post cq failed, err = %d\n", err);577 578 return err;579}580 581/* drain and optionally complete the send queue582 * if unable to complete a wqe, i.e. cq is full, stop583 * completing and flush the remaining wqes584 */585static void flush_send_queue(struct rxe_qp *qp, bool notify)586{587 struct rxe_send_wqe *wqe;588 struct rxe_queue *q = qp->sq.queue;589 int err;590 591 /* send queue never got created. nothing to do. */592 if (!qp->sq.queue)593 return;594 595 while ((wqe = queue_head(q, q->type))) {596 if (notify) {597 err = flush_send_wqe(qp, wqe);598 if (err)599 notify = 0;600 }601 queue_advance_consumer(q, q->type);602 }603}604 605static void free_pkt(struct rxe_pkt_info *pkt)606{607 struct sk_buff *skb = PKT_TO_SKB(pkt);608 struct rxe_qp *qp = pkt->qp;609 struct ib_device *dev = qp->ibqp.device;610 611 kfree_skb(skb);612 rxe_put(qp);613 ib_device_put(dev);614}615 616/* reset the retry timer if617 * - QP is type RC618 * - there is a packet sent by the requester that619 * might be acked (we still might get spurious620 * timeouts but try to keep them as few as possible)621 * - the timeout parameter is set622 * - the QP is alive623 */624static void reset_retry_timer(struct rxe_qp *qp)625{626 unsigned long flags;627 628 if (qp_type(qp) == IB_QPT_RC && qp->qp_timeout_jiffies) {629 spin_lock_irqsave(&qp->state_lock, flags);630 if (qp_state(qp) >= IB_QPS_RTS &&631 psn_compare(qp->req.psn, qp->comp.psn) > 0)632 mod_timer(&qp->retrans_timer,633 jiffies + qp->qp_timeout_jiffies);634 spin_unlock_irqrestore(&qp->state_lock, flags);635 }636}637 638int rxe_completer(struct rxe_qp *qp)639{640 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);641 struct rxe_send_wqe *wqe = NULL;642 struct sk_buff *skb = NULL;643 struct rxe_pkt_info *pkt = NULL;644 enum comp_state state;645 int ret;646 unsigned long flags;647 648 qp->req.again = 0;649 650 spin_lock_irqsave(&qp->state_lock, flags);651 if (!qp->valid || qp_state(qp) == IB_QPS_ERR ||652 qp_state(qp) == IB_QPS_RESET) {653 bool notify = qp->valid && (qp_state(qp) == IB_QPS_ERR);654 655 drain_resp_pkts(qp);656 flush_send_queue(qp, notify);657 spin_unlock_irqrestore(&qp->state_lock, flags);658 goto exit;659 }660 spin_unlock_irqrestore(&qp->state_lock, flags);661 662 if (qp->comp.timeout) {663 qp->comp.timeout_retry = 1;664 qp->comp.timeout = 0;665 } else {666 qp->comp.timeout_retry = 0;667 }668 669 if (qp->req.need_retry)670 goto exit;671 672 state = COMPST_GET_ACK;673 674 while (1) {675 rxe_dbg_qp(qp, "state = %s\n", comp_state_name[state]);676 switch (state) {677 case COMPST_GET_ACK:678 skb = skb_dequeue(&qp->resp_pkts);679 if (skb) {680 pkt = SKB_TO_PKT(skb);681 qp->comp.timeout_retry = 0;682 }683 state = COMPST_GET_WQE;684 break;685 686 case COMPST_GET_WQE:687 state = get_wqe(qp, pkt, &wqe);688 break;689 690 case COMPST_CHECK_PSN:691 state = check_psn(qp, pkt, wqe);692 break;693 694 case COMPST_CHECK_ACK:695 state = check_ack(qp, pkt, wqe);696 break;697 698 case COMPST_READ:699 state = do_read(qp, pkt, wqe);700 break;701 702 case COMPST_ATOMIC:703 state = do_atomic(qp, pkt, wqe);704 break;705 706 case COMPST_WRITE_SEND:707 if (wqe->state == wqe_state_pending &&708 wqe->last_psn == pkt->psn)709 state = COMPST_COMP_ACK;710 else711 state = COMPST_UPDATE_COMP;712 break;713 714 case COMPST_COMP_ACK:715 state = complete_ack(qp, pkt, wqe);716 break;717 718 case COMPST_COMP_WQE:719 state = complete_wqe(qp, pkt, wqe);720 break;721 722 case COMPST_UPDATE_COMP:723 if (pkt->mask & RXE_END_MASK)724 qp->comp.opcode = -1;725 else726 qp->comp.opcode = pkt->opcode;727 728 if (psn_compare(pkt->psn, qp->comp.psn) >= 0)729 qp->comp.psn = (pkt->psn + 1) & BTH_PSN_MASK;730 731 if (qp->req.wait_psn) {732 qp->req.wait_psn = 0;733 qp->req.again = 1;734 }735 736 state = COMPST_DONE;737 break;738 739 case COMPST_DONE:740 goto done;741 742 case COMPST_EXIT:743 if (qp->comp.timeout_retry && wqe) {744 state = COMPST_ERROR_RETRY;745 break;746 }747 748 reset_retry_timer(qp);749 goto exit;750 751 case COMPST_ERROR_RETRY:752 /* we come here if the retry timer fired and we did753 * not receive a response packet. try to retry the send754 * queue if that makes sense and the limits have not755 * been exceeded. remember that some timeouts are756 * spurious since we do not reset the timer but kick757 * it down the road or let it expire758 */759 760 /* there is nothing to retry in this case */761 if (!wqe || (wqe->state == wqe_state_posted))762 goto exit;763 764 /* if we've started a retry, don't start another765 * retry sequence, unless this is a timeout.766 */767 if (qp->comp.started_retry &&768 !qp->comp.timeout_retry)769 goto done;770 771 if (qp->comp.retry_cnt > 0) {772 if (qp->comp.retry_cnt != 7)773 qp->comp.retry_cnt--;774 775 /* no point in retrying if we have already776 * seen the last ack that the requester could777 * have caused778 */779 if (psn_compare(qp->req.psn,780 qp->comp.psn) > 0) {781 /* tell the requester to retry the782 * send queue next time around783 */784 rxe_counter_inc(rxe,785 RXE_CNT_COMP_RETRY);786 qp->req.need_retry = 1;787 qp->comp.started_retry = 1;788 qp->req.again = 1;789 }790 goto done;791 792 } else {793 rxe_counter_inc(rxe, RXE_CNT_RETRY_EXCEEDED);794 wqe->status = IB_WC_RETRY_EXC_ERR;795 state = COMPST_ERROR;796 }797 break;798 799 case COMPST_RNR_RETRY:800 /* we come here if we received an RNR NAK */801 if (qp->comp.rnr_retry > 0) {802 if (qp->comp.rnr_retry != 7)803 qp->comp.rnr_retry--;804 805 /* don't start a retry flow until the806 * rnr timer has fired807 */808 qp->req.wait_for_rnr_timer = 1;809 rxe_dbg_qp(qp, "set rnr nak timer\n");810 // TODO who protects from destroy_qp??811 mod_timer(&qp->rnr_nak_timer,812 jiffies + rnrnak_jiffies(aeth_syn(pkt)813 & ~AETH_TYPE_MASK));814 goto exit;815 } else {816 rxe_counter_inc(rxe,817 RXE_CNT_RNR_RETRY_EXCEEDED);818 wqe->status = IB_WC_RNR_RETRY_EXC_ERR;819 state = COMPST_ERROR;820 }821 break;822 823 case COMPST_ERROR:824 WARN_ON_ONCE(wqe->status == IB_WC_SUCCESS);825 do_complete(qp, wqe);826 rxe_qp_error(qp);827 goto exit;828 }829 }830 831 /* A non-zero return value will cause rxe_do_task to832 * exit its loop and end the work item. A zero return833 * will continue looping and return to rxe_completer834 */835done:836 ret = 0;837 goto out;838exit:839 ret = (qp->req.again) ? 0 : -EAGAIN;840out:841 qp->req.again = 0;842 if (pkt)843 free_pkt(pkt);844 return ret;845}846