1304 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause2 3/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */4/* Copyright (c) 2008-2019, IBM Corporation */5 6#include <linux/errno.h>7#include <linux/types.h>8#include <linux/net.h>9#include <linux/scatterlist.h>10#include <linux/highmem.h>11#include <net/tcp.h>12 13#include <rdma/iw_cm.h>14#include <rdma/ib_verbs.h>15#include <rdma/ib_user_verbs.h>16 17#include "siw.h"18#include "siw_verbs.h"19#include "siw_mem.h"20 21#define MAX_HDR_INLINE \22 (((uint32_t)(sizeof(struct siw_rreq_pkt) - \23 sizeof(struct iwarp_send))) & 0xF8)24 25static struct page *siw_get_pblpage(struct siw_mem *mem, u64 addr, int *idx)26{27 struct siw_pbl *pbl = mem->pbl;28 u64 offset = addr - mem->va;29 dma_addr_t paddr = siw_pbl_get_buffer(pbl, offset, NULL, idx);30 31 if (paddr)32 return ib_virt_dma_to_page(paddr);33 34 return NULL;35}36 37static struct page *siw_get_page(struct siw_mem *mem, struct siw_sge *sge,38 unsigned long offset, int *pbl_idx)39{40 if (!mem->is_pbl)41 return siw_get_upage(mem->umem, sge->laddr + offset);42 else43 return siw_get_pblpage(mem, sge->laddr + offset, pbl_idx);44}45 46/*47 * Copy short payload at provided destination payload address48 */49static int siw_try_1seg(struct siw_iwarp_tx *c_tx, void *paddr)50{51 struct siw_wqe *wqe = &c_tx->wqe_active;52 struct siw_sge *sge = &wqe->sqe.sge[0];53 u32 bytes = sge->length;54 55 if (bytes > MAX_HDR_INLINE || wqe->sqe.num_sge != 1)56 return MAX_HDR_INLINE + 1;57 58 if (!bytes)59 return 0;60 61 if (tx_flags(wqe) & SIW_WQE_INLINE) {62 memcpy(paddr, &wqe->sqe.sge[1], bytes);63 } else {64 struct siw_mem *mem = wqe->mem[0];65 66 if (!mem->mem_obj) {67 /* Kernel client using kva */68 memcpy(paddr, ib_virt_dma_to_ptr(sge->laddr), bytes);69 } else if (c_tx->in_syscall) {70 if (copy_from_user(paddr, u64_to_user_ptr(sge->laddr),71 bytes))72 return -EFAULT;73 } else {74 unsigned int off = sge->laddr & ~PAGE_MASK;75 struct page *p;76 char *buffer;77 int pbl_idx = 0;78 79 p = siw_get_page(mem, sge, 0, &pbl_idx);80 if (unlikely(!p))81 return -EFAULT;82 83 buffer = kmap_local_page(p);84 85 if (likely(PAGE_SIZE - off >= bytes)) {86 memcpy(paddr, buffer + off, bytes);87 } else {88 unsigned long part = bytes - (PAGE_SIZE - off);89 90 memcpy(paddr, buffer + off, part);91 kunmap_local(buffer);92 93 p = siw_get_page(mem, sge, part, &pbl_idx);94 if (unlikely(!p))95 return -EFAULT;96 97 buffer = kmap_local_page(p);98 memcpy(paddr + part, buffer, bytes - part);99 }100 kunmap_local(buffer);101 }102 }103 return (int)bytes;104}105 106#define PKT_FRAGMENTED 1107#define PKT_COMPLETE 0108 109/*110 * siw_qp_prepare_tx()111 *112 * Prepare tx state for sending out one fpdu. Builds complete pkt113 * if no user data or only immediate data are present.114 *115 * returns PKT_COMPLETE if complete pkt built, PKT_FRAGMENTED otherwise.116 */117static int siw_qp_prepare_tx(struct siw_iwarp_tx *c_tx)118{119 struct siw_wqe *wqe = &c_tx->wqe_active;120 char *crc = NULL;121 int data = 0;122 123 switch (tx_type(wqe)) {124 case SIW_OP_READ:125 case SIW_OP_READ_LOCAL_INV:126 memcpy(&c_tx->pkt.ctrl,127 &iwarp_pktinfo[RDMAP_RDMA_READ_REQ].ctrl,128 sizeof(struct iwarp_ctrl));129 130 c_tx->pkt.rreq.rsvd = 0;131 c_tx->pkt.rreq.ddp_qn = htonl(RDMAP_UNTAGGED_QN_RDMA_READ);132 c_tx->pkt.rreq.ddp_msn =133 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ]);134 c_tx->pkt.rreq.ddp_mo = 0;135 c_tx->pkt.rreq.sink_stag = htonl(wqe->sqe.sge[0].lkey);136 c_tx->pkt.rreq.sink_to =137 cpu_to_be64(wqe->sqe.sge[0].laddr);138 c_tx->pkt.rreq.source_stag = htonl(wqe->sqe.rkey);139 c_tx->pkt.rreq.source_to = cpu_to_be64(wqe->sqe.raddr);140 c_tx->pkt.rreq.read_size = htonl(wqe->sqe.sge[0].length);141 142 c_tx->ctrl_len = sizeof(struct iwarp_rdma_rreq);143 crc = (char *)&c_tx->pkt.rreq_pkt.crc;144 break;145 146 case SIW_OP_SEND:147 if (tx_flags(wqe) & SIW_WQE_SOLICITED)148 memcpy(&c_tx->pkt.ctrl,149 &iwarp_pktinfo[RDMAP_SEND_SE].ctrl,150 sizeof(struct iwarp_ctrl));151 else152 memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_SEND].ctrl,153 sizeof(struct iwarp_ctrl));154 155 c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND;156 c_tx->pkt.send.ddp_msn =157 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]);158 c_tx->pkt.send.ddp_mo = 0;159 160 c_tx->pkt.send_inv.inval_stag = 0;161 162 c_tx->ctrl_len = sizeof(struct iwarp_send);163 164 crc = (char *)&c_tx->pkt.send_pkt.crc;165 data = siw_try_1seg(c_tx, crc);166 break;167 168 case SIW_OP_SEND_REMOTE_INV:169 if (tx_flags(wqe) & SIW_WQE_SOLICITED)170 memcpy(&c_tx->pkt.ctrl,171 &iwarp_pktinfo[RDMAP_SEND_SE_INVAL].ctrl,172 sizeof(struct iwarp_ctrl));173 else174 memcpy(&c_tx->pkt.ctrl,175 &iwarp_pktinfo[RDMAP_SEND_INVAL].ctrl,176 sizeof(struct iwarp_ctrl));177 178 c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND;179 c_tx->pkt.send.ddp_msn =180 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]);181 c_tx->pkt.send.ddp_mo = 0;182 183 c_tx->pkt.send_inv.inval_stag = cpu_to_be32(wqe->sqe.rkey);184 185 c_tx->ctrl_len = sizeof(struct iwarp_send_inv);186 187 crc = (char *)&c_tx->pkt.send_pkt.crc;188 data = siw_try_1seg(c_tx, crc);189 break;190 191 case SIW_OP_WRITE:192 memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_RDMA_WRITE].ctrl,193 sizeof(struct iwarp_ctrl));194 195 c_tx->pkt.rwrite.sink_stag = htonl(wqe->sqe.rkey);196 c_tx->pkt.rwrite.sink_to = cpu_to_be64(wqe->sqe.raddr);197 c_tx->ctrl_len = sizeof(struct iwarp_rdma_write);198 199 crc = (char *)&c_tx->pkt.write_pkt.crc;200 data = siw_try_1seg(c_tx, crc);201 break;202 203 case SIW_OP_READ_RESPONSE:204 memcpy(&c_tx->pkt.ctrl,205 &iwarp_pktinfo[RDMAP_RDMA_READ_RESP].ctrl,206 sizeof(struct iwarp_ctrl));207 208 /* NBO */209 c_tx->pkt.rresp.sink_stag = cpu_to_be32(wqe->sqe.rkey);210 c_tx->pkt.rresp.sink_to = cpu_to_be64(wqe->sqe.raddr);211 212 c_tx->ctrl_len = sizeof(struct iwarp_rdma_rresp);213 214 crc = (char *)&c_tx->pkt.write_pkt.crc;215 data = siw_try_1seg(c_tx, crc);216 break;217 218 default:219 siw_dbg_qp(tx_qp(c_tx), "stale wqe type %d\n", tx_type(wqe));220 return -EOPNOTSUPP;221 }222 if (unlikely(data < 0))223 return data;224 225 c_tx->ctrl_sent = 0;226 227 if (data <= MAX_HDR_INLINE) {228 if (data) {229 wqe->processed = data;230 231 c_tx->pkt.ctrl.mpa_len =232 htons(c_tx->ctrl_len + data - MPA_HDR_SIZE);233 234 /* Add pad, if needed */235 data += -(int)data & 0x3;236 /* advance CRC location after payload */237 crc += data;238 c_tx->ctrl_len += data;239 240 if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED))241 c_tx->pkt.c_untagged.ddp_mo = 0;242 else243 c_tx->pkt.c_tagged.ddp_to =244 cpu_to_be64(wqe->sqe.raddr);245 }246 247 *(u32 *)crc = 0;248 /*249 * Do complete CRC if enabled and short packet250 */251 if (c_tx->mpa_crc_hd &&252 crypto_shash_digest(c_tx->mpa_crc_hd, (u8 *)&c_tx->pkt,253 c_tx->ctrl_len, (u8 *)crc) != 0)254 return -EINVAL;255 c_tx->ctrl_len += MPA_CRC_SIZE;256 257 return PKT_COMPLETE;258 }259 c_tx->ctrl_len += MPA_CRC_SIZE;260 c_tx->sge_idx = 0;261 c_tx->sge_off = 0;262 c_tx->pbl_idx = 0;263 264 /*265 * Allow direct sending out of user buffer if WR is non signalled266 * and payload is over threshold.267 * Per RDMA verbs, the application should not change the send buffer268 * until the work completed. In iWarp, work completion is only269 * local delivery to TCP. TCP may reuse the buffer for270 * retransmission. Changing unsent data also breaks the CRC,271 * if applied.272 */273 if (c_tx->zcopy_tx && wqe->bytes >= SENDPAGE_THRESH &&274 !(tx_flags(wqe) & SIW_WQE_SIGNALLED))275 c_tx->use_sendpage = 1;276 else277 c_tx->use_sendpage = 0;278 279 return PKT_FRAGMENTED;280}281 282/*283 * Send out one complete control type FPDU, or header of FPDU carrying284 * data. Used for fixed sized packets like Read.Requests or zero length285 * SENDs, WRITEs, READ.Responses, or header only.286 */287static int siw_tx_ctrl(struct siw_iwarp_tx *c_tx, struct socket *s,288 int flags)289{290 struct msghdr msg = { .msg_flags = flags };291 struct kvec iov = { .iov_base =292 (char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent,293 .iov_len = c_tx->ctrl_len - c_tx->ctrl_sent };294 295 int rv = kernel_sendmsg(s, &msg, &iov, 1, iov.iov_len);296 297 if (rv >= 0) {298 c_tx->ctrl_sent += rv;299 300 if (c_tx->ctrl_sent == c_tx->ctrl_len)301 rv = 0;302 else303 rv = -EAGAIN;304 }305 return rv;306}307 308/*309 * 0copy TCP transmit interface: Use MSG_SPLICE_PAGES.310 *311 * Using sendpage to push page by page appears to be less efficient312 * than using sendmsg, even if data are copied.313 *314 * A general performance limitation might be the extra four bytes315 * trailer checksum segment to be pushed after user data.316 */317static int siw_tcp_sendpages(struct socket *s, struct page **page, int offset,318 size_t size)319{320 struct bio_vec bvec;321 struct msghdr msg = {322 .msg_flags = (MSG_MORE | MSG_DONTWAIT | MSG_SPLICE_PAGES),323 };324 struct sock *sk = s->sk;325 int i = 0, rv = 0, sent = 0;326 327 while (size) {328 size_t bytes = min_t(size_t, PAGE_SIZE - offset, size);329 330 if (size + offset <= PAGE_SIZE)331 msg.msg_flags &= ~MSG_MORE;332 333 tcp_rate_check_app_limited(sk);334 if (!sendpage_ok(page[i]))335 msg.msg_flags &= ~MSG_SPLICE_PAGES;336 bvec_set_page(&bvec, page[i], bytes, offset);337 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, size);338 339try_page_again:340 lock_sock(sk);341 rv = tcp_sendmsg_locked(sk, &msg, size);342 release_sock(sk);343 344 if (rv > 0) {345 size -= rv;346 sent += rv;347 if (rv != bytes) {348 offset += rv;349 bytes -= rv;350 goto try_page_again;351 }352 offset = 0;353 } else {354 if (rv == -EAGAIN || rv == 0)355 break;356 return rv;357 }358 i++;359 }360 return sent;361}362 363/*364 * siw_0copy_tx()365 *366 * Pushes list of pages to TCP socket. If pages from multiple367 * SGE's, all referenced pages of each SGE are pushed in one368 * shot.369 */370static int siw_0copy_tx(struct socket *s, struct page **page,371 struct siw_sge *sge, unsigned int offset,372 unsigned int size)373{374 int i = 0, sent = 0, rv;375 int sge_bytes = min(sge->length - offset, size);376 377 offset = (sge->laddr + offset) & ~PAGE_MASK;378 379 while (sent != size) {380 rv = siw_tcp_sendpages(s, &page[i], offset, sge_bytes);381 if (rv >= 0) {382 sent += rv;383 if (size == sent || sge_bytes > rv)384 break;385 386 i += PAGE_ALIGN(sge_bytes + offset) >> PAGE_SHIFT;387 sge++;388 sge_bytes = min(sge->length, size - sent);389 offset = sge->laddr & ~PAGE_MASK;390 } else {391 sent = rv;392 break;393 }394 }395 return sent;396}397 398#define MAX_TRAILER (MPA_CRC_SIZE + 4)399 400static void siw_unmap_pages(struct kvec *iov, unsigned long kmap_mask, int len)401{402 int i;403 404 /*405 * Work backwards through the array to honor the kmap_local_page()406 * ordering requirements.407 */408 for (i = (len-1); i >= 0; i--) {409 if (kmap_mask & BIT(i)) {410 unsigned long addr = (unsigned long)iov[i].iov_base;411 412 kunmap_local((void *)(addr & PAGE_MASK));413 }414 }415}416 417/*418 * siw_tx_hdt() tries to push a complete packet to TCP where all419 * packet fragments are referenced by the elements of one iovec.420 * For the data portion, each involved page must be referenced by421 * one extra element. All sge's data can be non-aligned to page422 * boundaries. Two more elements are referencing iWARP header423 * and trailer:424 * MAX_ARRAY = 64KB/PAGE_SIZE + 1 + (2 * (SIW_MAX_SGE - 1) + HDR + TRL425 */426#define MAX_ARRAY ((0xffff / PAGE_SIZE) + 1 + (2 * (SIW_MAX_SGE - 1) + 2))427 428/*429 * Write out iov referencing hdr, data and trailer of current FPDU.430 * Update transmit state dependent on write return status431 */432static int siw_tx_hdt(struct siw_iwarp_tx *c_tx, struct socket *s)433{434 struct siw_wqe *wqe = &c_tx->wqe_active;435 struct siw_sge *sge = &wqe->sqe.sge[c_tx->sge_idx];436 struct kvec iov[MAX_ARRAY];437 struct page *page_array[MAX_ARRAY];438 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR };439 440 int seg = 0, do_crc = c_tx->do_crc, is_kva = 0, rv;441 unsigned int data_len = c_tx->bytes_unsent, hdr_len = 0, trl_len = 0,442 sge_off = c_tx->sge_off, sge_idx = c_tx->sge_idx,443 pbl_idx = c_tx->pbl_idx;444 unsigned long kmap_mask = 0L;445 446 if (c_tx->state == SIW_SEND_HDR) {447 if (c_tx->use_sendpage) {448 rv = siw_tx_ctrl(c_tx, s, MSG_DONTWAIT | MSG_MORE);449 if (rv)450 goto done;451 452 c_tx->state = SIW_SEND_DATA;453 } else {454 iov[0].iov_base =455 (char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent;456 iov[0].iov_len = hdr_len =457 c_tx->ctrl_len - c_tx->ctrl_sent;458 seg = 1;459 }460 }461 462 wqe->processed += data_len;463 464 while (data_len) { /* walk the list of SGE's */465 unsigned int sge_len = min(sge->length - sge_off, data_len);466 unsigned int fp_off = (sge->laddr + sge_off) & ~PAGE_MASK;467 struct siw_mem *mem;468 469 if (!(tx_flags(wqe) & SIW_WQE_INLINE)) {470 mem = wqe->mem[sge_idx];471 is_kva = mem->mem_obj == NULL ? 1 : 0;472 } else {473 is_kva = 1;474 }475 if (is_kva && !c_tx->use_sendpage) {476 /*477 * tx from kernel virtual address: either inline data478 * or memory region with assigned kernel buffer479 */480 iov[seg].iov_base =481 ib_virt_dma_to_ptr(sge->laddr + sge_off);482 iov[seg].iov_len = sge_len;483 484 if (do_crc)485 crypto_shash_update(c_tx->mpa_crc_hd,486 iov[seg].iov_base,487 sge_len);488 sge_off += sge_len;489 data_len -= sge_len;490 seg++;491 goto sge_done;492 }493 494 while (sge_len) {495 size_t plen = min((int)PAGE_SIZE - fp_off, sge_len);496 void *kaddr;497 498 if (!is_kva) {499 struct page *p;500 501 p = siw_get_page(mem, sge, sge_off, &pbl_idx);502 if (unlikely(!p)) {503 siw_unmap_pages(iov, kmap_mask, seg);504 wqe->processed -= c_tx->bytes_unsent;505 rv = -EFAULT;506 goto done_crc;507 }508 page_array[seg] = p;509 510 if (!c_tx->use_sendpage) {511 void *kaddr = kmap_local_page(p);512 513 /* Remember for later kunmap() */514 kmap_mask |= BIT(seg);515 iov[seg].iov_base = kaddr + fp_off;516 iov[seg].iov_len = plen;517 518 if (do_crc)519 crypto_shash_update(520 c_tx->mpa_crc_hd,521 iov[seg].iov_base,522 plen);523 } else if (do_crc) {524 kaddr = kmap_local_page(p);525 crypto_shash_update(c_tx->mpa_crc_hd,526 kaddr + fp_off,527 plen);528 kunmap_local(kaddr);529 }530 } else {531 /*532 * Cast to an uintptr_t to preserve all 64 bits533 * in sge->laddr.534 */535 u64 va = sge->laddr + sge_off;536 537 page_array[seg] = ib_virt_dma_to_page(va);538 if (do_crc)539 crypto_shash_update(540 c_tx->mpa_crc_hd,541 ib_virt_dma_to_ptr(va),542 plen);543 }544 545 sge_len -= plen;546 sge_off += plen;547 data_len -= plen;548 fp_off = 0;549 550 if (++seg >= (int)MAX_ARRAY) {551 siw_dbg_qp(tx_qp(c_tx), "to many fragments\n");552 siw_unmap_pages(iov, kmap_mask, seg-1);553 wqe->processed -= c_tx->bytes_unsent;554 rv = -EMSGSIZE;555 goto done_crc;556 }557 }558sge_done:559 /* Update SGE variables at end of SGE */560 if (sge_off == sge->length &&561 (data_len != 0 || wqe->processed < wqe->bytes)) {562 sge_idx++;563 sge++;564 sge_off = 0;565 }566 }567 /* trailer */568 if (likely(c_tx->state != SIW_SEND_TRAILER)) {569 iov[seg].iov_base = &c_tx->trailer.pad[4 - c_tx->pad];570 iov[seg].iov_len = trl_len = MAX_TRAILER - (4 - c_tx->pad);571 } else {572 iov[seg].iov_base = &c_tx->trailer.pad[c_tx->ctrl_sent];573 iov[seg].iov_len = trl_len = MAX_TRAILER - c_tx->ctrl_sent;574 }575 576 if (c_tx->pad) {577 *(u32 *)c_tx->trailer.pad = 0;578 if (do_crc)579 crypto_shash_update(c_tx->mpa_crc_hd,580 (u8 *)&c_tx->trailer.crc - c_tx->pad,581 c_tx->pad);582 }583 if (!c_tx->mpa_crc_hd)584 c_tx->trailer.crc = 0;585 else if (do_crc)586 crypto_shash_final(c_tx->mpa_crc_hd, (u8 *)&c_tx->trailer.crc);587 588 data_len = c_tx->bytes_unsent;589 590 if (c_tx->use_sendpage) {591 rv = siw_0copy_tx(s, page_array, &wqe->sqe.sge[c_tx->sge_idx],592 c_tx->sge_off, data_len);593 if (rv == data_len) {594 rv = kernel_sendmsg(s, &msg, &iov[seg], 1, trl_len);595 if (rv > 0)596 rv += data_len;597 else598 rv = data_len;599 }600 } else {601 rv = kernel_sendmsg(s, &msg, iov, seg + 1,602 hdr_len + data_len + trl_len);603 siw_unmap_pages(iov, kmap_mask, seg);604 }605 if (rv < (int)hdr_len) {606 /* Not even complete hdr pushed or negative rv */607 wqe->processed -= data_len;608 if (rv >= 0) {609 c_tx->ctrl_sent += rv;610 rv = -EAGAIN;611 }612 goto done_crc;613 }614 rv -= hdr_len;615 616 if (rv >= (int)data_len) {617 /* all user data pushed to TCP or no data to push */618 if (data_len > 0 && wqe->processed < wqe->bytes) {619 /* Save the current state for next tx */620 c_tx->sge_idx = sge_idx;621 c_tx->sge_off = sge_off;622 c_tx->pbl_idx = pbl_idx;623 }624 rv -= data_len;625 626 if (rv == trl_len) /* all pushed */627 rv = 0;628 else {629 c_tx->state = SIW_SEND_TRAILER;630 c_tx->ctrl_len = MAX_TRAILER;631 c_tx->ctrl_sent = rv + 4 - c_tx->pad;632 c_tx->bytes_unsent = 0;633 rv = -EAGAIN;634 }635 636 } else if (data_len > 0) {637 /* Maybe some user data pushed to TCP */638 c_tx->state = SIW_SEND_DATA;639 wqe->processed -= data_len - rv;640 641 if (rv) {642 /*643 * Some bytes out. Recompute tx state based644 * on old state and bytes pushed645 */646 unsigned int sge_unsent;647 648 c_tx->bytes_unsent -= rv;649 sge = &wqe->sqe.sge[c_tx->sge_idx];650 sge_unsent = sge->length - c_tx->sge_off;651 652 while (sge_unsent <= rv) {653 rv -= sge_unsent;654 c_tx->sge_idx++;655 c_tx->sge_off = 0;656 sge++;657 sge_unsent = sge->length;658 }659 c_tx->sge_off += rv;660 }661 rv = -EAGAIN;662 }663done_crc:664 c_tx->do_crc = 0;665done:666 return rv;667}668 669static void siw_update_tcpseg(struct siw_iwarp_tx *c_tx,670 struct socket *s)671{672 struct tcp_sock *tp = tcp_sk(s->sk);673 674 if (tp->gso_segs) {675 if (c_tx->gso_seg_limit == 0)676 c_tx->tcp_seglen = tp->mss_cache * tp->gso_segs;677 else678 c_tx->tcp_seglen =679 tp->mss_cache *680 min_t(u16, c_tx->gso_seg_limit, tp->gso_segs);681 } else {682 c_tx->tcp_seglen = tp->mss_cache;683 }684 /* Loopback may give odd numbers */685 c_tx->tcp_seglen &= 0xfffffff8;686}687 688/*689 * siw_prepare_fpdu()690 *691 * Prepares transmit context to send out one FPDU if FPDU will contain692 * user data and user data are not immediate data.693 * Computes maximum FPDU length to fill up TCP MSS if possible.694 *695 * @qp: QP from which to transmit696 * @wqe: Current WQE causing transmission697 *698 * TODO: Take into account real available sendspace on socket699 * to avoid header misalignment due to send pausing within700 * fpdu transmission701 */702static void siw_prepare_fpdu(struct siw_qp *qp, struct siw_wqe *wqe)703{704 struct siw_iwarp_tx *c_tx = &qp->tx_ctx;705 int data_len;706 707 c_tx->ctrl_len =708 iwarp_pktinfo[__rdmap_get_opcode(&c_tx->pkt.ctrl)].hdr_len;709 c_tx->ctrl_sent = 0;710 711 /*712 * Update target buffer offset if any713 */714 if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED))715 /* Untagged message */716 c_tx->pkt.c_untagged.ddp_mo = cpu_to_be32(wqe->processed);717 else /* Tagged message */718 c_tx->pkt.c_tagged.ddp_to =719 cpu_to_be64(wqe->sqe.raddr + wqe->processed);720 721 data_len = wqe->bytes - wqe->processed;722 if (data_len + c_tx->ctrl_len + MPA_CRC_SIZE > c_tx->tcp_seglen) {723 /* Trim DDP payload to fit into current TCP segment */724 data_len = c_tx->tcp_seglen - (c_tx->ctrl_len + MPA_CRC_SIZE);725 c_tx->pkt.ctrl.ddp_rdmap_ctrl &= ~DDP_FLAG_LAST;726 c_tx->pad = 0;727 } else {728 c_tx->pkt.ctrl.ddp_rdmap_ctrl |= DDP_FLAG_LAST;729 c_tx->pad = -data_len & 0x3;730 }731 c_tx->bytes_unsent = data_len;732 733 c_tx->pkt.ctrl.mpa_len =734 htons(c_tx->ctrl_len + data_len - MPA_HDR_SIZE);735 736 /*737 * Init MPA CRC computation738 */739 if (c_tx->mpa_crc_hd) {740 crypto_shash_init(c_tx->mpa_crc_hd);741 crypto_shash_update(c_tx->mpa_crc_hd, (u8 *)&c_tx->pkt,742 c_tx->ctrl_len);743 c_tx->do_crc = 1;744 }745}746 747/*748 * siw_check_sgl_tx()749 *750 * Check permissions for a list of SGE's (SGL).751 * A successful check will have all memory referenced752 * for transmission resolved and assigned to the WQE.753 *754 * @pd: Protection Domain SGL should belong to755 * @wqe: WQE to be checked756 * @perms: requested access permissions757 *758 */759 760static int siw_check_sgl_tx(struct ib_pd *pd, struct siw_wqe *wqe,761 enum ib_access_flags perms)762{763 struct siw_sge *sge = &wqe->sqe.sge[0];764 int i, len, num_sge = wqe->sqe.num_sge;765 766 if (unlikely(num_sge > SIW_MAX_SGE))767 return -EINVAL;768 769 for (i = 0, len = 0; num_sge; num_sge--, i++, sge++) {770 /*771 * rdma verbs: do not check stag for a zero length sge772 */773 if (sge->length) {774 int rv = siw_check_sge(pd, sge, &wqe->mem[i], perms, 0,775 sge->length);776 777 if (unlikely(rv != E_ACCESS_OK))778 return rv;779 }780 len += sge->length;781 }782 return len;783}784 785/*786 * siw_qp_sq_proc_tx()787 *788 * Process one WQE which needs transmission on the wire.789 */790static int siw_qp_sq_proc_tx(struct siw_qp *qp, struct siw_wqe *wqe)791{792 struct siw_iwarp_tx *c_tx = &qp->tx_ctx;793 struct socket *s = qp->attrs.sk;794 int rv = 0, burst_len = qp->tx_ctx.burst;795 enum rdmap_ecode ecode = RDMAP_ECODE_CATASTROPHIC_STREAM;796 797 if (unlikely(wqe->wr_status == SIW_WR_IDLE))798 return 0;799 800 if (!burst_len)801 burst_len = SQ_USER_MAXBURST;802 803 if (wqe->wr_status == SIW_WR_QUEUED) {804 if (!(wqe->sqe.flags & SIW_WQE_INLINE)) {805 if (tx_type(wqe) == SIW_OP_READ_RESPONSE)806 wqe->sqe.num_sge = 1;807 808 if (tx_type(wqe) != SIW_OP_READ &&809 tx_type(wqe) != SIW_OP_READ_LOCAL_INV) {810 /*811 * Reference memory to be tx'd w/o checking812 * access for LOCAL_READ permission, since813 * not defined in RDMA core.814 */815 rv = siw_check_sgl_tx(qp->pd, wqe, 0);816 if (rv < 0) {817 if (tx_type(wqe) ==818 SIW_OP_READ_RESPONSE)819 ecode = siw_rdmap_error(-rv);820 rv = -EINVAL;821 goto tx_error;822 }823 wqe->bytes = rv;824 } else {825 wqe->bytes = 0;826 }827 } else {828 wqe->bytes = wqe->sqe.sge[0].length;829 if (!rdma_is_kernel_res(&qp->base_qp.res)) {830 if (wqe->bytes > SIW_MAX_INLINE) {831 rv = -EINVAL;832 goto tx_error;833 }834 wqe->sqe.sge[0].laddr =835 (u64)(uintptr_t)&wqe->sqe.sge[1];836 }837 }838 wqe->wr_status = SIW_WR_INPROGRESS;839 wqe->processed = 0;840 841 siw_update_tcpseg(c_tx, s);842 843 rv = siw_qp_prepare_tx(c_tx);844 if (rv == PKT_FRAGMENTED) {845 c_tx->state = SIW_SEND_HDR;846 siw_prepare_fpdu(qp, wqe);847 } else if (rv == PKT_COMPLETE) {848 c_tx->state = SIW_SEND_SHORT_FPDU;849 } else {850 goto tx_error;851 }852 }853 854next_segment:855 siw_dbg_qp(qp, "wr type %d, state %d, data %u, sent %u, id %llx\n",856 tx_type(wqe), wqe->wr_status, wqe->bytes, wqe->processed,857 wqe->sqe.id);858 859 if (--burst_len == 0) {860 rv = -EINPROGRESS;861 goto tx_done;862 }863 if (c_tx->state == SIW_SEND_SHORT_FPDU) {864 enum siw_opcode tx_type = tx_type(wqe);865 unsigned int msg_flags;866 867 if (siw_sq_empty(qp) || !siw_tcp_nagle || burst_len == 1)868 /*869 * End current TCP segment, if SQ runs empty,870 * or siw_tcp_nagle is not set, or we bail out871 * soon due to no burst credit left.872 */873 msg_flags = MSG_DONTWAIT;874 else875 msg_flags = MSG_DONTWAIT | MSG_MORE;876 877 rv = siw_tx_ctrl(c_tx, s, msg_flags);878 879 if (!rv && tx_type != SIW_OP_READ &&880 tx_type != SIW_OP_READ_LOCAL_INV)881 wqe->processed = wqe->bytes;882 883 goto tx_done;884 885 } else {886 rv = siw_tx_hdt(c_tx, s);887 }888 if (!rv) {889 /*890 * One segment sent. Processing completed if last891 * segment, Do next segment otherwise.892 */893 if (unlikely(c_tx->tx_suspend)) {894 /*895 * Verbs, 6.4.: Try stopping sending after a full896 * DDP segment if the connection goes down897 * (== peer halfclose)898 */899 rv = -ECONNABORTED;900 goto tx_done;901 }902 if (c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_LAST) {903 siw_dbg_qp(qp, "WQE completed\n");904 goto tx_done;905 }906 c_tx->state = SIW_SEND_HDR;907 908 siw_update_tcpseg(c_tx, s);909 910 siw_prepare_fpdu(qp, wqe);911 goto next_segment;912 }913tx_done:914 qp->tx_ctx.burst = burst_len;915 return rv;916 917tx_error:918 if (ecode != RDMAP_ECODE_CATASTROPHIC_STREAM)919 siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP,920 RDMAP_ETYPE_REMOTE_PROTECTION, ecode, 1);921 else922 siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP,923 RDMAP_ETYPE_CATASTROPHIC,924 RDMAP_ECODE_UNSPECIFIED, 1);925 return rv;926}927 928static int siw_fastreg_mr(struct ib_pd *pd, struct siw_sqe *sqe)929{930 struct ib_mr *base_mr = (struct ib_mr *)(uintptr_t)sqe->base_mr;931 struct siw_device *sdev = to_siw_dev(pd->device);932 struct siw_mem *mem;933 int rv = 0;934 935 siw_dbg_pd(pd, "STag 0x%08x\n", sqe->rkey);936 937 if (unlikely(!base_mr)) {938 pr_warn("siw: fastreg: STag 0x%08x unknown\n", sqe->rkey);939 return -EINVAL;940 }941 942 if (unlikely(base_mr->rkey >> 8 != sqe->rkey >> 8)) {943 pr_warn("siw: fastreg: STag 0x%08x: bad MR\n", sqe->rkey);944 return -EINVAL;945 }946 947 mem = siw_mem_id2obj(sdev, sqe->rkey >> 8);948 if (unlikely(!mem)) {949 pr_warn("siw: fastreg: STag 0x%08x unknown\n", sqe->rkey);950 return -EINVAL;951 }952 953 if (unlikely(mem->pd != pd)) {954 pr_warn("siw: fastreg: PD mismatch\n");955 rv = -EINVAL;956 goto out;957 }958 if (unlikely(mem->stag_valid)) {959 pr_warn("siw: fastreg: STag 0x%08x already valid\n", sqe->rkey);960 rv = -EINVAL;961 goto out;962 }963 /* Refresh STag since user may have changed key part */964 mem->stag = sqe->rkey;965 mem->perms = sqe->access;966 967 siw_dbg_mem(mem, "STag 0x%08x now valid\n", sqe->rkey);968 mem->va = base_mr->iova;969 mem->stag_valid = 1;970out:971 siw_mem_put(mem);972 return rv;973}974 975static int siw_qp_sq_proc_local(struct siw_qp *qp, struct siw_wqe *wqe)976{977 int rv;978 979 switch (tx_type(wqe)) {980 case SIW_OP_REG_MR:981 rv = siw_fastreg_mr(qp->pd, &wqe->sqe);982 break;983 984 case SIW_OP_INVAL_STAG:985 rv = siw_invalidate_stag(qp->pd, wqe->sqe.rkey);986 break;987 988 default:989 rv = -EINVAL;990 }991 return rv;992}993 994/*995 * siw_qp_sq_process()996 *997 * Core TX path routine for RDMAP/DDP/MPA using a TCP kernel socket.998 * Sends RDMAP payload for the current SQ WR @wqe of @qp in one or more999 * MPA FPDUs, each containing a DDP segment.1000 *1001 * SQ processing may occur in user context as a result of posting1002 * new WQE's or from siw_tx_thread context. Processing in1003 * user context is limited to non-kernel verbs users.1004 *1005 * SQ processing may get paused anytime, possibly in the middle of a WR1006 * or FPDU, if insufficient send space is available. SQ processing1007 * gets resumed from siw_tx_thread, if send space becomes available again.1008 *1009 * Must be called with the QP state read-locked.1010 *1011 * Note:1012 * An outbound RREQ can be satisfied by the corresponding RRESP1013 * _before_ it gets assigned to the ORQ. This happens regularly1014 * in RDMA READ via loopback case. Since both outbound RREQ and1015 * inbound RRESP can be handled by the same CPU, locking the ORQ1016 * is dead-lock prone and thus not an option. With that, the1017 * RREQ gets assigned to the ORQ _before_ being sent - see1018 * siw_activate_tx() - and pulled back in case of send failure.1019 */1020int siw_qp_sq_process(struct siw_qp *qp)1021{1022 struct siw_wqe *wqe = tx_wqe(qp);1023 enum siw_opcode tx_type;1024 unsigned long flags;1025 int rv = 0;1026 1027 siw_dbg_qp(qp, "enter for type %d\n", tx_type(wqe));1028 1029next_wqe:1030 /*1031 * Stop QP processing if SQ state changed1032 */1033 if (unlikely(qp->tx_ctx.tx_suspend)) {1034 siw_dbg_qp(qp, "tx suspended\n");1035 goto done;1036 }1037 tx_type = tx_type(wqe);1038 1039 if (tx_type <= SIW_OP_READ_RESPONSE)1040 rv = siw_qp_sq_proc_tx(qp, wqe);1041 else1042 rv = siw_qp_sq_proc_local(qp, wqe);1043 1044 if (!rv) {1045 /*1046 * WQE processing done1047 */1048 switch (tx_type) {1049 case SIW_OP_SEND:1050 case SIW_OP_SEND_REMOTE_INV:1051 case SIW_OP_WRITE:1052 siw_wqe_put_mem(wqe, tx_type);1053 fallthrough;1054 1055 case SIW_OP_INVAL_STAG:1056 case SIW_OP_REG_MR:1057 if (tx_flags(wqe) & SIW_WQE_SIGNALLED)1058 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,1059 SIW_WC_SUCCESS);1060 break;1061 1062 case SIW_OP_READ:1063 case SIW_OP_READ_LOCAL_INV:1064 /*1065 * already enqueued to ORQ queue1066 */1067 break;1068 1069 case SIW_OP_READ_RESPONSE:1070 siw_wqe_put_mem(wqe, tx_type);1071 break;1072 1073 default:1074 WARN(1, "undefined WQE type %d\n", tx_type);1075 rv = -EINVAL;1076 goto done;1077 }1078 1079 spin_lock_irqsave(&qp->sq_lock, flags);1080 wqe->wr_status = SIW_WR_IDLE;1081 rv = siw_activate_tx(qp);1082 spin_unlock_irqrestore(&qp->sq_lock, flags);1083 1084 if (rv <= 0)1085 goto done;1086 1087 goto next_wqe;1088 1089 } else if (rv == -EAGAIN) {1090 siw_dbg_qp(qp, "sq paused: hd/tr %d of %d, data %d\n",1091 qp->tx_ctx.ctrl_sent, qp->tx_ctx.ctrl_len,1092 qp->tx_ctx.bytes_unsent);1093 rv = 0;1094 goto done;1095 } else if (rv == -EINPROGRESS) {1096 rv = siw_sq_start(qp);1097 goto done;1098 } else {1099 /*1100 * WQE processing failed.1101 * Verbs 8.3.2:1102 * o It turns any WQE into a signalled WQE.1103 * o Local catastrophic error must be surfaced1104 * o QP must be moved into Terminate state: done by code1105 * doing socket state change processing1106 *1107 * o TODO: Termination message must be sent.1108 * o TODO: Implement more precise work completion errors,1109 * see enum ib_wc_status in ib_verbs.h1110 */1111 siw_dbg_qp(qp, "wqe type %d processing failed: %d\n",1112 tx_type(wqe), rv);1113 1114 spin_lock_irqsave(&qp->sq_lock, flags);1115 /*1116 * RREQ may have already been completed by inbound RRESP!1117 */1118 if ((tx_type == SIW_OP_READ ||1119 tx_type == SIW_OP_READ_LOCAL_INV) && qp->attrs.orq_size) {1120 /* Cleanup pending entry in ORQ */1121 qp->orq_put--;1122 qp->orq[qp->orq_put % qp->attrs.orq_size].flags = 0;1123 }1124 spin_unlock_irqrestore(&qp->sq_lock, flags);1125 /*1126 * immediately suspends further TX processing1127 */1128 if (!qp->tx_ctx.tx_suspend)1129 siw_qp_cm_drop(qp, 0);1130 1131 switch (tx_type) {1132 case SIW_OP_SEND:1133 case SIW_OP_SEND_REMOTE_INV:1134 case SIW_OP_SEND_WITH_IMM:1135 case SIW_OP_WRITE:1136 case SIW_OP_READ:1137 case SIW_OP_READ_LOCAL_INV:1138 siw_wqe_put_mem(wqe, tx_type);1139 fallthrough;1140 1141 case SIW_OP_INVAL_STAG:1142 case SIW_OP_REG_MR:1143 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,1144 SIW_WC_LOC_QP_OP_ERR);1145 1146 siw_qp_event(qp, IB_EVENT_QP_FATAL);1147 1148 break;1149 1150 case SIW_OP_READ_RESPONSE:1151 siw_dbg_qp(qp, "proc. read.response failed: %d\n", rv);1152 1153 siw_qp_event(qp, IB_EVENT_QP_REQ_ERR);1154 1155 siw_wqe_put_mem(wqe, SIW_OP_READ_RESPONSE);1156 1157 break;1158 1159 default:1160 WARN(1, "undefined WQE type %d\n", tx_type);1161 rv = -EINVAL;1162 }1163 wqe->wr_status = SIW_WR_IDLE;1164 }1165done:1166 return rv;1167}1168 1169static void siw_sq_resume(struct siw_qp *qp)1170{1171 if (down_read_trylock(&qp->state_lock)) {1172 if (likely(qp->attrs.state == SIW_QP_STATE_RTS &&1173 !qp->tx_ctx.tx_suspend)) {1174 int rv = siw_qp_sq_process(qp);1175 1176 up_read(&qp->state_lock);1177 1178 if (unlikely(rv < 0)) {1179 siw_dbg_qp(qp, "SQ task failed: err %d\n", rv);1180 1181 if (!qp->tx_ctx.tx_suspend)1182 siw_qp_cm_drop(qp, 0);1183 }1184 } else {1185 up_read(&qp->state_lock);1186 }1187 } else {1188 siw_dbg_qp(qp, "Resume SQ while QP locked\n");1189 }1190 siw_qp_put(qp);1191}1192 1193struct tx_task_t {1194 struct llist_head active;1195 wait_queue_head_t waiting;1196};1197 1198static DEFINE_PER_CPU(struct tx_task_t, siw_tx_task_g);1199 1200int siw_create_tx_threads(void)1201{1202 int cpu, assigned = 0;1203 1204 for_each_online_cpu(cpu) {1205 struct tx_task_t *tx_task;1206 1207 /* Skip HT cores */1208 if (cpu % cpumask_weight(topology_sibling_cpumask(cpu)))1209 continue;1210 1211 tx_task = &per_cpu(siw_tx_task_g, cpu);1212 init_llist_head(&tx_task->active);1213 init_waitqueue_head(&tx_task->waiting);1214 1215 siw_tx_thread[cpu] =1216 kthread_run_on_cpu(siw_run_sq,1217 (unsigned long *)(long)cpu,1218 cpu, "siw_tx/%u");1219 if (IS_ERR(siw_tx_thread[cpu])) {1220 siw_tx_thread[cpu] = NULL;1221 continue;1222 }1223 assigned++;1224 }1225 return assigned;1226}1227 1228void siw_stop_tx_threads(void)1229{1230 int cpu;1231 1232 for_each_possible_cpu(cpu) {1233 if (siw_tx_thread[cpu]) {1234 kthread_stop(siw_tx_thread[cpu]);1235 wake_up(&per_cpu(siw_tx_task_g, cpu).waiting);1236 siw_tx_thread[cpu] = NULL;1237 }1238 }1239}1240 1241int siw_run_sq(void *data)1242{1243 const int nr_cpu = (unsigned int)(long)data;1244 struct llist_node *active;1245 struct siw_qp *qp;1246 struct tx_task_t *tx_task = &per_cpu(siw_tx_task_g, nr_cpu);1247 1248 while (1) {1249 struct llist_node *fifo_list = NULL;1250 1251 wait_event_interruptible(tx_task->waiting,1252 !llist_empty(&tx_task->active) ||1253 kthread_should_stop());1254 1255 if (kthread_should_stop())1256 break;1257 1258 active = llist_del_all(&tx_task->active);1259 /*1260 * llist_del_all returns a list with newest entry first.1261 * Re-order list for fairness among QP's.1262 */1263 fifo_list = llist_reverse_order(active);1264 while (fifo_list) {1265 qp = container_of(fifo_list, struct siw_qp, tx_list);1266 fifo_list = llist_next(fifo_list);1267 qp->tx_list.next = NULL;1268 1269 siw_sq_resume(qp);1270 }1271 }1272 active = llist_del_all(&tx_task->active);1273 if (active) {1274 llist_for_each_entry(qp, active, tx_list) {1275 qp->tx_list.next = NULL;1276 siw_sq_resume(qp);1277 }1278 }1279 return 0;1280}1281 1282int siw_sq_start(struct siw_qp *qp)1283{1284 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE)1285 return 0;1286 1287 if (unlikely(!cpu_online(qp->tx_cpu))) {1288 siw_put_tx_cpu(qp->tx_cpu);1289 qp->tx_cpu = siw_get_tx_cpu(qp->sdev);1290 if (qp->tx_cpu < 0) {1291 pr_warn("siw: no tx cpu available\n");1292 1293 return -EIO;1294 }1295 }1296 siw_qp_get(qp);1297 1298 llist_add(&qp->tx_list, &per_cpu(siw_tx_task_g, qp->tx_cpu).active);1299 1300 wake_up(&per_cpu(siw_tx_task_g, qp->tx_cpu).waiting);1301 1302 return 0;1303}1304