522 lines · c
1/*2 * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.3 * All rights reserved.4 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.5 *6 * This software is available to you under a choice of one of two7 * licenses. You may choose to be licensed under the terms of the GNU8 * General Public License (GPL) Version 2, available from the file9 * COPYING in the main directory of this source tree, or the10 * OpenIB.org BSD license below:11 *12 * Redistribution and use in source and binary forms, with or13 * without modification, are permitted provided that the following14 * conditions are met:15 *16 * - Redistributions of source code must retain the above17 * copyright notice, this list of conditions and the following18 * disclaimer.19 *20 * - Redistributions in binary form must reproduce the above21 * copyright notice, this list of conditions and the following22 * disclaimer in the documentation and/or other materials23 * provided with the distribution.24 *25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE32 * SOFTWARE.33 */34 35#include "qib.h"36 37/* cut down ridiculously long IB macro names */38#define OP(x) IB_OPCODE_UC_##x39 40/**41 * qib_make_uc_req - construct a request packet (SEND, RDMA write)42 * @qp: a pointer to the QP43 * @flags: unused44 *45 * Assumes the s_lock is held.46 *47 * Return 1 if constructed; otherwise, return 0.48 */49int qib_make_uc_req(struct rvt_qp *qp, unsigned long *flags)50{51 struct qib_qp_priv *priv = qp->priv;52 struct ib_other_headers *ohdr;53 struct rvt_swqe *wqe;54 u32 hwords;55 u32 bth0;56 u32 len;57 u32 pmtu = qp->pmtu;58 int ret = 0;59 60 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) {61 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))62 goto bail;63 /* We are in the error state, flush the work request. */64 if (qp->s_last == READ_ONCE(qp->s_head))65 goto bail;66 /* If DMAs are in progress, we can't flush immediately. */67 if (atomic_read(&priv->s_dma_busy)) {68 qp->s_flags |= RVT_S_WAIT_DMA;69 goto bail;70 }71 wqe = rvt_get_swqe_ptr(qp, qp->s_last);72 rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);73 goto done;74 }75 76 ohdr = &priv->s_hdr->u.oth;77 if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)78 ohdr = &priv->s_hdr->u.l.oth;79 80 /* header size in 32-bit words LRH+BTH = (8+12)/4. */81 hwords = 5;82 bth0 = 0;83 84 /* Get the next send request. */85 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);86 qp->s_wqe = NULL;87 switch (qp->s_state) {88 default:89 if (!(ib_rvt_state_ops[qp->state] &90 RVT_PROCESS_NEXT_SEND_OK))91 goto bail;92 /* Check if send work queue is empty. */93 if (qp->s_cur == READ_ONCE(qp->s_head))94 goto bail;95 /*96 * Start a new request.97 */98 qp->s_psn = wqe->psn;99 qp->s_sge.sge = wqe->sg_list[0];100 qp->s_sge.sg_list = wqe->sg_list + 1;101 qp->s_sge.num_sge = wqe->wr.num_sge;102 qp->s_sge.total_len = wqe->length;103 len = wqe->length;104 qp->s_len = len;105 switch (wqe->wr.opcode) {106 case IB_WR_SEND:107 case IB_WR_SEND_WITH_IMM:108 if (len > pmtu) {109 qp->s_state = OP(SEND_FIRST);110 len = pmtu;111 break;112 }113 if (wqe->wr.opcode == IB_WR_SEND)114 qp->s_state = OP(SEND_ONLY);115 else {116 qp->s_state =117 OP(SEND_ONLY_WITH_IMMEDIATE);118 /* Immediate data comes after the BTH */119 ohdr->u.imm_data = wqe->wr.ex.imm_data;120 hwords += 1;121 }122 if (wqe->wr.send_flags & IB_SEND_SOLICITED)123 bth0 |= IB_BTH_SOLICITED;124 qp->s_wqe = wqe;125 if (++qp->s_cur >= qp->s_size)126 qp->s_cur = 0;127 break;128 129 case IB_WR_RDMA_WRITE:130 case IB_WR_RDMA_WRITE_WITH_IMM:131 ohdr->u.rc.reth.vaddr =132 cpu_to_be64(wqe->rdma_wr.remote_addr);133 ohdr->u.rc.reth.rkey =134 cpu_to_be32(wqe->rdma_wr.rkey);135 ohdr->u.rc.reth.length = cpu_to_be32(len);136 hwords += sizeof(struct ib_reth) / 4;137 if (len > pmtu) {138 qp->s_state = OP(RDMA_WRITE_FIRST);139 len = pmtu;140 break;141 }142 if (wqe->wr.opcode == IB_WR_RDMA_WRITE)143 qp->s_state = OP(RDMA_WRITE_ONLY);144 else {145 qp->s_state =146 OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE);147 /* Immediate data comes after the RETH */148 ohdr->u.rc.imm_data = wqe->wr.ex.imm_data;149 hwords += 1;150 if (wqe->wr.send_flags & IB_SEND_SOLICITED)151 bth0 |= IB_BTH_SOLICITED;152 }153 qp->s_wqe = wqe;154 if (++qp->s_cur >= qp->s_size)155 qp->s_cur = 0;156 break;157 158 default:159 goto bail;160 }161 break;162 163 case OP(SEND_FIRST):164 qp->s_state = OP(SEND_MIDDLE);165 fallthrough;166 case OP(SEND_MIDDLE):167 len = qp->s_len;168 if (len > pmtu) {169 len = pmtu;170 break;171 }172 if (wqe->wr.opcode == IB_WR_SEND)173 qp->s_state = OP(SEND_LAST);174 else {175 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);176 /* Immediate data comes after the BTH */177 ohdr->u.imm_data = wqe->wr.ex.imm_data;178 hwords += 1;179 }180 if (wqe->wr.send_flags & IB_SEND_SOLICITED)181 bth0 |= IB_BTH_SOLICITED;182 qp->s_wqe = wqe;183 if (++qp->s_cur >= qp->s_size)184 qp->s_cur = 0;185 break;186 187 case OP(RDMA_WRITE_FIRST):188 qp->s_state = OP(RDMA_WRITE_MIDDLE);189 fallthrough;190 case OP(RDMA_WRITE_MIDDLE):191 len = qp->s_len;192 if (len > pmtu) {193 len = pmtu;194 break;195 }196 if (wqe->wr.opcode == IB_WR_RDMA_WRITE)197 qp->s_state = OP(RDMA_WRITE_LAST);198 else {199 qp->s_state =200 OP(RDMA_WRITE_LAST_WITH_IMMEDIATE);201 /* Immediate data comes after the BTH */202 ohdr->u.imm_data = wqe->wr.ex.imm_data;203 hwords += 1;204 if (wqe->wr.send_flags & IB_SEND_SOLICITED)205 bth0 |= IB_BTH_SOLICITED;206 }207 qp->s_wqe = wqe;208 if (++qp->s_cur >= qp->s_size)209 qp->s_cur = 0;210 break;211 }212 qp->s_len -= len;213 qp->s_hdrwords = hwords;214 qp->s_cur_sge = &qp->s_sge;215 qp->s_cur_size = len;216 qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24),217 qp->s_psn++ & QIB_PSN_MASK);218done:219 return 1;220bail:221 qp->s_flags &= ~RVT_S_BUSY;222 return ret;223}224 225/**226 * qib_uc_rcv - handle an incoming UC packet227 * @ibp: the port the packet came in on228 * @hdr: the header of the packet229 * @has_grh: true if the packet has a GRH230 * @data: the packet data231 * @tlen: the length of the packet232 * @qp: the QP for this packet.233 *234 * This is called from qib_qp_rcv() to process an incoming UC packet235 * for the given QP.236 * Called at interrupt level.237 */238void qib_uc_rcv(struct qib_ibport *ibp, struct ib_header *hdr,239 int has_grh, void *data, u32 tlen, struct rvt_qp *qp)240{241 struct ib_other_headers *ohdr;242 u32 opcode;243 u32 hdrsize;244 u32 psn;245 u32 pad;246 struct ib_wc wc;247 u32 pmtu = qp->pmtu;248 struct ib_reth *reth;249 int ret;250 251 /* Check for GRH */252 if (!has_grh) {253 ohdr = &hdr->u.oth;254 hdrsize = 8 + 12; /* LRH + BTH */255 } else {256 ohdr = &hdr->u.l.oth;257 hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */258 }259 260 opcode = be32_to_cpu(ohdr->bth[0]);261 if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode))262 return;263 264 psn = be32_to_cpu(ohdr->bth[2]);265 opcode >>= 24;266 267 /* Compare the PSN verses the expected PSN. */268 if (unlikely(qib_cmp24(psn, qp->r_psn) != 0)) {269 /*270 * Handle a sequence error.271 * Silently drop any current message.272 */273 qp->r_psn = psn;274inv:275 if (qp->r_state == OP(SEND_FIRST) ||276 qp->r_state == OP(SEND_MIDDLE)) {277 set_bit(RVT_R_REWIND_SGE, &qp->r_aflags);278 qp->r_sge.num_sge = 0;279 } else280 rvt_put_ss(&qp->r_sge);281 qp->r_state = OP(SEND_LAST);282 switch (opcode) {283 case OP(SEND_FIRST):284 case OP(SEND_ONLY):285 case OP(SEND_ONLY_WITH_IMMEDIATE):286 goto send_first;287 288 case OP(RDMA_WRITE_FIRST):289 case OP(RDMA_WRITE_ONLY):290 case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE):291 goto rdma_first;292 293 default:294 goto drop;295 }296 }297 298 /* Check for opcode sequence errors. */299 switch (qp->r_state) {300 case OP(SEND_FIRST):301 case OP(SEND_MIDDLE):302 if (opcode == OP(SEND_MIDDLE) ||303 opcode == OP(SEND_LAST) ||304 opcode == OP(SEND_LAST_WITH_IMMEDIATE))305 break;306 goto inv;307 308 case OP(RDMA_WRITE_FIRST):309 case OP(RDMA_WRITE_MIDDLE):310 if (opcode == OP(RDMA_WRITE_MIDDLE) ||311 opcode == OP(RDMA_WRITE_LAST) ||312 opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))313 break;314 goto inv;315 316 default:317 if (opcode == OP(SEND_FIRST) ||318 opcode == OP(SEND_ONLY) ||319 opcode == OP(SEND_ONLY_WITH_IMMEDIATE) ||320 opcode == OP(RDMA_WRITE_FIRST) ||321 opcode == OP(RDMA_WRITE_ONLY) ||322 opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE))323 break;324 goto inv;325 }326 327 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST))328 rvt_comm_est(qp);329 330 /* OK, process the packet. */331 switch (opcode) {332 case OP(SEND_FIRST):333 case OP(SEND_ONLY):334 case OP(SEND_ONLY_WITH_IMMEDIATE):335send_first:336 if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags))337 qp->r_sge = qp->s_rdma_read_sge;338 else {339 ret = rvt_get_rwqe(qp, false);340 if (ret < 0)341 goto op_err;342 if (!ret)343 goto drop;344 /*345 * qp->s_rdma_read_sge will be the owner346 * of the mr references.347 */348 qp->s_rdma_read_sge = qp->r_sge;349 }350 qp->r_rcv_len = 0;351 if (opcode == OP(SEND_ONLY))352 goto no_immediate_data;353 else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE))354 goto send_last_imm;355 fallthrough;356 case OP(SEND_MIDDLE):357 /* Check for invalid length PMTU or posted rwqe len. */358 if (unlikely(tlen != (hdrsize + pmtu + 4)))359 goto rewind;360 qp->r_rcv_len += pmtu;361 if (unlikely(qp->r_rcv_len > qp->r_len))362 goto rewind;363 rvt_copy_sge(qp, &qp->r_sge, data, pmtu, false, false);364 break;365 366 case OP(SEND_LAST_WITH_IMMEDIATE):367send_last_imm:368 wc.ex.imm_data = ohdr->u.imm_data;369 hdrsize += 4;370 wc.wc_flags = IB_WC_WITH_IMM;371 goto send_last;372 case OP(SEND_LAST):373no_immediate_data:374 wc.ex.imm_data = 0;375 wc.wc_flags = 0;376send_last:377 /* Get the number of bytes the message was padded by. */378 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;379 /* Check for invalid length. */380 /* XXX LAST len should be >= 1 */381 if (unlikely(tlen < (hdrsize + pad + 4)))382 goto rewind;383 /* Don't count the CRC. */384 tlen -= (hdrsize + pad + 4);385 wc.byte_len = tlen + qp->r_rcv_len;386 if (unlikely(wc.byte_len > qp->r_len))387 goto rewind;388 wc.opcode = IB_WC_RECV;389 rvt_copy_sge(qp, &qp->r_sge, data, tlen, false, false);390 rvt_put_ss(&qp->s_rdma_read_sge);391last_imm:392 wc.wr_id = qp->r_wr_id;393 wc.status = IB_WC_SUCCESS;394 wc.qp = &qp->ibqp;395 wc.src_qp = qp->remote_qpn;396 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr);397 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr);398 /* zero fields that are N/A */399 wc.vendor_err = 0;400 wc.pkey_index = 0;401 wc.dlid_path_bits = 0;402 wc.port_num = 0;403 /* Signal completion event if the solicited bit is set. */404 rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr));405 break;406 407 case OP(RDMA_WRITE_FIRST):408 case OP(RDMA_WRITE_ONLY):409 case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): /* consume RWQE */410rdma_first:411 if (unlikely(!(qp->qp_access_flags &412 IB_ACCESS_REMOTE_WRITE))) {413 goto drop;414 }415 reth = &ohdr->u.rc.reth;416 hdrsize += sizeof(*reth);417 qp->r_len = be32_to_cpu(reth->length);418 qp->r_rcv_len = 0;419 qp->r_sge.sg_list = NULL;420 if (qp->r_len != 0) {421 u32 rkey = be32_to_cpu(reth->rkey);422 u64 vaddr = be64_to_cpu(reth->vaddr);423 int ok;424 425 /* Check rkey */426 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len,427 vaddr, rkey, IB_ACCESS_REMOTE_WRITE);428 if (unlikely(!ok))429 goto drop;430 qp->r_sge.num_sge = 1;431 } else {432 qp->r_sge.num_sge = 0;433 qp->r_sge.sge.mr = NULL;434 qp->r_sge.sge.vaddr = NULL;435 qp->r_sge.sge.length = 0;436 qp->r_sge.sge.sge_length = 0;437 }438 if (opcode == OP(RDMA_WRITE_ONLY))439 goto rdma_last;440 else if (opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) {441 wc.ex.imm_data = ohdr->u.rc.imm_data;442 goto rdma_last_imm;443 }444 fallthrough;445 case OP(RDMA_WRITE_MIDDLE):446 /* Check for invalid length PMTU or posted rwqe len. */447 if (unlikely(tlen != (hdrsize + pmtu + 4)))448 goto drop;449 qp->r_rcv_len += pmtu;450 if (unlikely(qp->r_rcv_len > qp->r_len))451 goto drop;452 rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false);453 break;454 455 case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):456 wc.ex.imm_data = ohdr->u.imm_data;457rdma_last_imm:458 hdrsize += 4;459 wc.wc_flags = IB_WC_WITH_IMM;460 461 /* Get the number of bytes the message was padded by. */462 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;463 /* Check for invalid length. */464 /* XXX LAST len should be >= 1 */465 if (unlikely(tlen < (hdrsize + pad + 4)))466 goto drop;467 /* Don't count the CRC. */468 tlen -= (hdrsize + pad + 4);469 if (unlikely(tlen + qp->r_rcv_len != qp->r_len))470 goto drop;471 if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags))472 rvt_put_ss(&qp->s_rdma_read_sge);473 else {474 ret = rvt_get_rwqe(qp, true);475 if (ret < 0)476 goto op_err;477 if (!ret)478 goto drop;479 }480 wc.byte_len = qp->r_len;481 wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;482 rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false);483 rvt_put_ss(&qp->r_sge);484 goto last_imm;485 486 case OP(RDMA_WRITE_LAST):487rdma_last:488 /* Get the number of bytes the message was padded by. */489 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;490 /* Check for invalid length. */491 /* XXX LAST len should be >= 1 */492 if (unlikely(tlen < (hdrsize + pad + 4)))493 goto drop;494 /* Don't count the CRC. */495 tlen -= (hdrsize + pad + 4);496 if (unlikely(tlen + qp->r_rcv_len != qp->r_len))497 goto drop;498 rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false);499 rvt_put_ss(&qp->r_sge);500 break;501 502 default:503 /* Drop packet for unknown opcodes. */504 goto drop;505 }506 qp->r_psn++;507 qp->r_state = opcode;508 return;509 510rewind:511 set_bit(RVT_R_REWIND_SGE, &qp->r_aflags);512 qp->r_sge.num_sge = 0;513drop:514 ibp->rvp.n_pkt_drops++;515 return;516 517op_err:518 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);519 return;520 521}522