4434 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB2/* Copyright (c) 2015 - 2021 Intel Corporation */3#include "main.h"4#include "trace.h"5 6static void irdma_cm_post_event(struct irdma_cm_event *event);7static void irdma_disconnect_worker(struct work_struct *work);8 9/**10 * irdma_free_sqbuf - put back puda buffer if refcount is 011 * @vsi: The VSI structure of the device12 * @bufp: puda buffer to free13 */14void irdma_free_sqbuf(struct irdma_sc_vsi *vsi, void *bufp)15{16 struct irdma_puda_buf *buf = bufp;17 struct irdma_puda_rsrc *ilq = vsi->ilq;18 19 if (refcount_dec_and_test(&buf->refcount))20 irdma_puda_ret_bufpool(ilq, buf);21}22 23/**24 * irdma_record_ird_ord - Record IRD/ORD passed in25 * @cm_node: connection's node26 * @conn_ird: connection IRD27 * @conn_ord: connection ORD28 */29static void irdma_record_ird_ord(struct irdma_cm_node *cm_node, u32 conn_ird,30 u32 conn_ord)31{32 if (conn_ird > cm_node->dev->hw_attrs.max_hw_ird)33 conn_ird = cm_node->dev->hw_attrs.max_hw_ird;34 35 if (conn_ord > cm_node->dev->hw_attrs.max_hw_ord)36 conn_ord = cm_node->dev->hw_attrs.max_hw_ord;37 else if (!conn_ord && cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO)38 conn_ord = 1;39 cm_node->ird_size = conn_ird;40 cm_node->ord_size = conn_ord;41}42 43/**44 * irdma_copy_ip_ntohl - copy IP address from network to host45 * @dst: IP address in host order46 * @src: IP address in network order (big endian)47 */48void irdma_copy_ip_ntohl(u32 *dst, __be32 *src)49{50 *dst++ = ntohl(*src++);51 *dst++ = ntohl(*src++);52 *dst++ = ntohl(*src++);53 *dst = ntohl(*src);54}55 56/**57 * irdma_copy_ip_htonl - copy IP address from host to network order58 * @dst: IP address in network order (big endian)59 * @src: IP address in host order60 */61void irdma_copy_ip_htonl(__be32 *dst, u32 *src)62{63 *dst++ = htonl(*src++);64 *dst++ = htonl(*src++);65 *dst++ = htonl(*src++);66 *dst = htonl(*src);67}68 69/**70 * irdma_get_addr_info71 * @cm_node: contains ip/tcp info72 * @cm_info: to get a copy of the cm_node ip/tcp info73 */74static void irdma_get_addr_info(struct irdma_cm_node *cm_node,75 struct irdma_cm_info *cm_info)76{77 memset(cm_info, 0, sizeof(*cm_info));78 cm_info->ipv4 = cm_node->ipv4;79 cm_info->vlan_id = cm_node->vlan_id;80 memcpy(cm_info->loc_addr, cm_node->loc_addr, sizeof(cm_info->loc_addr));81 memcpy(cm_info->rem_addr, cm_node->rem_addr, sizeof(cm_info->rem_addr));82 cm_info->loc_port = cm_node->loc_port;83 cm_info->rem_port = cm_node->rem_port;84}85 86/**87 * irdma_fill_sockaddr4 - fill in addr info for IPv4 connection88 * @cm_node: connection's node89 * @event: upper layer's cm event90 */91static inline void irdma_fill_sockaddr4(struct irdma_cm_node *cm_node,92 struct iw_cm_event *event)93{94 struct sockaddr_in *laddr = (struct sockaddr_in *)&event->local_addr;95 struct sockaddr_in *raddr = (struct sockaddr_in *)&event->remote_addr;96 97 laddr->sin_family = AF_INET;98 raddr->sin_family = AF_INET;99 100 laddr->sin_port = htons(cm_node->loc_port);101 raddr->sin_port = htons(cm_node->rem_port);102 103 laddr->sin_addr.s_addr = htonl(cm_node->loc_addr[0]);104 raddr->sin_addr.s_addr = htonl(cm_node->rem_addr[0]);105}106 107/**108 * irdma_fill_sockaddr6 - fill in addr info for IPv6 connection109 * @cm_node: connection's node110 * @event: upper layer's cm event111 */112static inline void irdma_fill_sockaddr6(struct irdma_cm_node *cm_node,113 struct iw_cm_event *event)114{115 struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&event->local_addr;116 struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)&event->remote_addr;117 118 laddr6->sin6_family = AF_INET6;119 raddr6->sin6_family = AF_INET6;120 121 laddr6->sin6_port = htons(cm_node->loc_port);122 raddr6->sin6_port = htons(cm_node->rem_port);123 124 irdma_copy_ip_htonl(laddr6->sin6_addr.in6_u.u6_addr32,125 cm_node->loc_addr);126 irdma_copy_ip_htonl(raddr6->sin6_addr.in6_u.u6_addr32,127 cm_node->rem_addr);128}129 130/**131 * irdma_get_cmevent_info - for cm event upcall132 * @cm_node: connection's node133 * @cm_id: upper layers cm struct for the event134 * @event: upper layer's cm event135 */136static inline void irdma_get_cmevent_info(struct irdma_cm_node *cm_node,137 struct iw_cm_id *cm_id,138 struct iw_cm_event *event)139{140 memcpy(&event->local_addr, &cm_id->m_local_addr,141 sizeof(event->local_addr));142 memcpy(&event->remote_addr, &cm_id->m_remote_addr,143 sizeof(event->remote_addr));144 if (cm_node) {145 event->private_data = cm_node->pdata_buf;146 event->private_data_len = (u8)cm_node->pdata.size;147 event->ird = cm_node->ird_size;148 event->ord = cm_node->ord_size;149 }150}151 152/**153 * irdma_send_cm_event - upcall cm's event handler154 * @cm_node: connection's node155 * @cm_id: upper layer's cm info struct156 * @type: Event type to indicate157 * @status: status for the event type158 */159static int irdma_send_cm_event(struct irdma_cm_node *cm_node,160 struct iw_cm_id *cm_id,161 enum iw_cm_event_type type, int status)162{163 struct iw_cm_event event = {};164 165 event.event = type;166 event.status = status;167 trace_irdma_send_cm_event(cm_node, cm_id, type, status,168 __builtin_return_address(0));169 170 ibdev_dbg(&cm_node->iwdev->ibdev,171 "CM: cm_node %p cm_id=%p state=%d accel=%d event_type=%d status=%d\n",172 cm_node, cm_id, cm_node->accelerated, cm_node->state, type,173 status);174 175 switch (type) {176 case IW_CM_EVENT_CONNECT_REQUEST:177 if (cm_node->ipv4)178 irdma_fill_sockaddr4(cm_node, &event);179 else180 irdma_fill_sockaddr6(cm_node, &event);181 event.provider_data = cm_node;182 event.private_data = cm_node->pdata_buf;183 event.private_data_len = (u8)cm_node->pdata.size;184 event.ird = cm_node->ird_size;185 break;186 case IW_CM_EVENT_CONNECT_REPLY:187 irdma_get_cmevent_info(cm_node, cm_id, &event);188 break;189 case IW_CM_EVENT_ESTABLISHED:190 event.ird = cm_node->ird_size;191 event.ord = cm_node->ord_size;192 break;193 case IW_CM_EVENT_DISCONNECT:194 case IW_CM_EVENT_CLOSE:195 /* Wait if we are in RTS but havent issued the iwcm event upcall */196 if (!cm_node->accelerated)197 wait_for_completion(&cm_node->establish_comp);198 break;199 default:200 return -EINVAL;201 }202 203 return cm_id->event_handler(cm_id, &event);204}205 206/**207 * irdma_timer_list_prep - add connection nodes to a list to perform timer tasks208 * @cm_core: cm's core209 * @timer_list: a timer list to which cm_node will be selected210 */211static void irdma_timer_list_prep(struct irdma_cm_core *cm_core,212 struct list_head *timer_list)213{214 struct irdma_cm_node *cm_node;215 int bkt;216 217 hash_for_each_rcu(cm_core->cm_hash_tbl, bkt, cm_node, list) {218 if ((cm_node->close_entry || cm_node->send_entry) &&219 refcount_inc_not_zero(&cm_node->refcnt))220 list_add(&cm_node->timer_entry, timer_list);221 }222}223 224/**225 * irdma_create_event - create cm event226 * @cm_node: connection's node227 * @type: Event type to generate228 */229static struct irdma_cm_event *irdma_create_event(struct irdma_cm_node *cm_node,230 enum irdma_cm_event_type type)231{232 struct irdma_cm_event *event;233 234 if (!cm_node->cm_id)235 return NULL;236 237 event = kzalloc(sizeof(*event), GFP_ATOMIC);238 239 if (!event)240 return NULL;241 242 event->type = type;243 event->cm_node = cm_node;244 memcpy(event->cm_info.rem_addr, cm_node->rem_addr,245 sizeof(event->cm_info.rem_addr));246 memcpy(event->cm_info.loc_addr, cm_node->loc_addr,247 sizeof(event->cm_info.loc_addr));248 event->cm_info.rem_port = cm_node->rem_port;249 event->cm_info.loc_port = cm_node->loc_port;250 event->cm_info.cm_id = cm_node->cm_id;251 ibdev_dbg(&cm_node->iwdev->ibdev,252 "CM: node=%p event=%p type=%u dst=%pI4 src=%pI4\n", cm_node,253 event, type, event->cm_info.loc_addr,254 event->cm_info.rem_addr);255 trace_irdma_create_event(cm_node, type, __builtin_return_address(0));256 irdma_cm_post_event(event);257 258 return event;259}260 261/**262 * irdma_free_retrans_entry - free send entry263 * @cm_node: connection's node264 */265static void irdma_free_retrans_entry(struct irdma_cm_node *cm_node)266{267 struct irdma_device *iwdev = cm_node->iwdev;268 struct irdma_timer_entry *send_entry;269 270 send_entry = cm_node->send_entry;271 if (!send_entry)272 return;273 274 cm_node->send_entry = NULL;275 irdma_free_sqbuf(&iwdev->vsi, send_entry->sqbuf);276 kfree(send_entry);277 refcount_dec(&cm_node->refcnt);278}279 280/**281 * irdma_cleanup_retrans_entry - free send entry with lock282 * @cm_node: connection's node283 */284static void irdma_cleanup_retrans_entry(struct irdma_cm_node *cm_node)285{286 unsigned long flags;287 288 spin_lock_irqsave(&cm_node->retrans_list_lock, flags);289 irdma_free_retrans_entry(cm_node);290 spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);291}292 293/**294 * irdma_form_ah_cm_frame - get a free packet and build frame with address handle295 * @cm_node: connection's node ionfo to use in frame296 * @options: pointer to options info297 * @hdr: pointer mpa header298 * @pdata: pointer to private data299 * @flags: indicates FIN or ACK300 */301static struct irdma_puda_buf *irdma_form_ah_cm_frame(struct irdma_cm_node *cm_node,302 struct irdma_kmem_info *options,303 struct irdma_kmem_info *hdr,304 struct irdma_mpa_priv_info *pdata,305 u8 flags)306{307 struct irdma_puda_buf *sqbuf;308 struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;309 u8 *buf;310 struct tcphdr *tcph;311 u16 pktsize;312 u32 opts_len = 0;313 u32 pd_len = 0;314 u32 hdr_len = 0;315 316 if (!cm_node->ah || !cm_node->ah->ah_info.ah_valid) {317 ibdev_dbg(&cm_node->iwdev->ibdev, "CM: AH invalid\n");318 return NULL;319 }320 321 sqbuf = irdma_puda_get_bufpool(vsi->ilq);322 if (!sqbuf) {323 ibdev_dbg(&cm_node->iwdev->ibdev, "CM: SQ buf NULL\n");324 return NULL;325 }326 327 sqbuf->ah_id = cm_node->ah->ah_info.ah_idx;328 buf = sqbuf->mem.va;329 if (options)330 opts_len = (u32)options->size;331 332 if (hdr)333 hdr_len = hdr->size;334 335 if (pdata)336 pd_len = pdata->size;337 338 pktsize = sizeof(*tcph) + opts_len + hdr_len + pd_len;339 340 memset(buf, 0, sizeof(*tcph));341 342 sqbuf->totallen = pktsize;343 sqbuf->tcphlen = sizeof(*tcph) + opts_len;344 sqbuf->scratch = cm_node;345 346 tcph = (struct tcphdr *)buf;347 buf += sizeof(*tcph);348 349 tcph->source = htons(cm_node->loc_port);350 tcph->dest = htons(cm_node->rem_port);351 tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num);352 353 if (flags & SET_ACK) {354 cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;355 tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num);356 tcph->ack = 1;357 } else {358 tcph->ack_seq = 0;359 }360 361 if (flags & SET_SYN) {362 cm_node->tcp_cntxt.loc_seq_num++;363 tcph->syn = 1;364 } else {365 cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len;366 }367 368 if (flags & SET_FIN) {369 cm_node->tcp_cntxt.loc_seq_num++;370 tcph->fin = 1;371 }372 373 if (flags & SET_RST)374 tcph->rst = 1;375 376 tcph->doff = (u16)((sizeof(*tcph) + opts_len + 3) >> 2);377 sqbuf->tcphlen = tcph->doff << 2;378 tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd);379 tcph->urg_ptr = 0;380 381 if (opts_len) {382 memcpy(buf, options->addr, opts_len);383 buf += opts_len;384 }385 386 if (hdr_len) {387 memcpy(buf, hdr->addr, hdr_len);388 buf += hdr_len;389 }390 391 if (pdata && pdata->addr)392 memcpy(buf, pdata->addr, pdata->size);393 394 refcount_set(&sqbuf->refcount, 1);395 396 print_hex_dump_debug("ILQ: TRANSMIT ILQ BUFFER", DUMP_PREFIX_OFFSET,397 16, 8, sqbuf->mem.va, sqbuf->totallen, false);398 399 return sqbuf;400}401 402/**403 * irdma_form_uda_cm_frame - get a free packet and build frame full tcpip packet404 * @cm_node: connection's node ionfo to use in frame405 * @options: pointer to options info406 * @hdr: pointer mpa header407 * @pdata: pointer to private data408 * @flags: indicates FIN or ACK409 */410static struct irdma_puda_buf *irdma_form_uda_cm_frame(struct irdma_cm_node *cm_node,411 struct irdma_kmem_info *options,412 struct irdma_kmem_info *hdr,413 struct irdma_mpa_priv_info *pdata,414 u8 flags)415{416 struct irdma_puda_buf *sqbuf;417 struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;418 u8 *buf;419 420 struct tcphdr *tcph;421 struct iphdr *iph;422 struct ipv6hdr *ip6h;423 struct ethhdr *ethh;424 u16 pktsize;425 u16 eth_hlen = ETH_HLEN;426 u32 opts_len = 0;427 u32 pd_len = 0;428 u32 hdr_len = 0;429 430 u16 vtag;431 432 sqbuf = irdma_puda_get_bufpool(vsi->ilq);433 if (!sqbuf)434 return NULL;435 436 buf = sqbuf->mem.va;437 438 if (options)439 opts_len = (u32)options->size;440 441 if (hdr)442 hdr_len = hdr->size;443 444 if (pdata)445 pd_len = pdata->size;446 447 if (cm_node->vlan_id < VLAN_N_VID)448 eth_hlen += 4;449 450 if (cm_node->ipv4)451 pktsize = sizeof(*iph) + sizeof(*tcph);452 else453 pktsize = sizeof(*ip6h) + sizeof(*tcph);454 pktsize += opts_len + hdr_len + pd_len;455 456 memset(buf, 0, eth_hlen + pktsize);457 458 sqbuf->totallen = pktsize + eth_hlen;459 sqbuf->maclen = eth_hlen;460 sqbuf->tcphlen = sizeof(*tcph) + opts_len;461 sqbuf->scratch = cm_node;462 463 ethh = (struct ethhdr *)buf;464 buf += eth_hlen;465 466 if (cm_node->do_lpb)467 sqbuf->do_lpb = true;468 469 if (cm_node->ipv4) {470 sqbuf->ipv4 = true;471 472 iph = (struct iphdr *)buf;473 buf += sizeof(*iph);474 tcph = (struct tcphdr *)buf;475 buf += sizeof(*tcph);476 477 ether_addr_copy(ethh->h_dest, cm_node->rem_mac);478 ether_addr_copy(ethh->h_source, cm_node->loc_mac);479 if (cm_node->vlan_id < VLAN_N_VID) {480 ((struct vlan_ethhdr *)ethh)->h_vlan_proto =481 htons(ETH_P_8021Q);482 vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) |483 cm_node->vlan_id;484 ((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);485 486 ((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto =487 htons(ETH_P_IP);488 } else {489 ethh->h_proto = htons(ETH_P_IP);490 }491 492 iph->version = IPVERSION;493 iph->ihl = 5; /* 5 * 4Byte words, IP headr len */494 iph->tos = cm_node->tos;495 iph->tot_len = htons(pktsize);496 iph->id = htons(++cm_node->tcp_cntxt.loc_id);497 498 iph->frag_off = htons(0x4000);499 iph->ttl = 0x40;500 iph->protocol = IPPROTO_TCP;501 iph->saddr = htonl(cm_node->loc_addr[0]);502 iph->daddr = htonl(cm_node->rem_addr[0]);503 } else {504 sqbuf->ipv4 = false;505 ip6h = (struct ipv6hdr *)buf;506 buf += sizeof(*ip6h);507 tcph = (struct tcphdr *)buf;508 buf += sizeof(*tcph);509 510 ether_addr_copy(ethh->h_dest, cm_node->rem_mac);511 ether_addr_copy(ethh->h_source, cm_node->loc_mac);512 if (cm_node->vlan_id < VLAN_N_VID) {513 ((struct vlan_ethhdr *)ethh)->h_vlan_proto =514 htons(ETH_P_8021Q);515 vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) |516 cm_node->vlan_id;517 ((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);518 ((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto =519 htons(ETH_P_IPV6);520 } else {521 ethh->h_proto = htons(ETH_P_IPV6);522 }523 ip6h->version = 6;524 ip6h->priority = cm_node->tos >> 4;525 ip6h->flow_lbl[0] = cm_node->tos << 4;526 ip6h->flow_lbl[1] = 0;527 ip6h->flow_lbl[2] = 0;528 ip6h->payload_len = htons(pktsize - sizeof(*ip6h));529 ip6h->nexthdr = 6;530 ip6h->hop_limit = 128;531 irdma_copy_ip_htonl(ip6h->saddr.in6_u.u6_addr32,532 cm_node->loc_addr);533 irdma_copy_ip_htonl(ip6h->daddr.in6_u.u6_addr32,534 cm_node->rem_addr);535 }536 537 tcph->source = htons(cm_node->loc_port);538 tcph->dest = htons(cm_node->rem_port);539 tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num);540 541 if (flags & SET_ACK) {542 cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;543 tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num);544 tcph->ack = 1;545 } else {546 tcph->ack_seq = 0;547 }548 549 if (flags & SET_SYN) {550 cm_node->tcp_cntxt.loc_seq_num++;551 tcph->syn = 1;552 } else {553 cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len;554 }555 556 if (flags & SET_FIN) {557 cm_node->tcp_cntxt.loc_seq_num++;558 tcph->fin = 1;559 }560 561 if (flags & SET_RST)562 tcph->rst = 1;563 564 tcph->doff = (u16)((sizeof(*tcph) + opts_len + 3) >> 2);565 sqbuf->tcphlen = tcph->doff << 2;566 tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd);567 tcph->urg_ptr = 0;568 569 if (opts_len) {570 memcpy(buf, options->addr, opts_len);571 buf += opts_len;572 }573 574 if (hdr_len) {575 memcpy(buf, hdr->addr, hdr_len);576 buf += hdr_len;577 }578 579 if (pdata && pdata->addr)580 memcpy(buf, pdata->addr, pdata->size);581 582 refcount_set(&sqbuf->refcount, 1);583 584 print_hex_dump_debug("ILQ: TRANSMIT ILQ BUFFER", DUMP_PREFIX_OFFSET,585 16, 8, sqbuf->mem.va, sqbuf->totallen, false);586 return sqbuf;587}588 589/**590 * irdma_send_reset - Send RST packet591 * @cm_node: connection's node592 */593int irdma_send_reset(struct irdma_cm_node *cm_node)594{595 struct irdma_puda_buf *sqbuf;596 int flags = SET_RST | SET_ACK;597 598 trace_irdma_send_reset(cm_node, 0, __builtin_return_address(0));599 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL,600 flags);601 if (!sqbuf)602 return -ENOMEM;603 604 ibdev_dbg(&cm_node->iwdev->ibdev,605 "CM: caller: %pS cm_node %p cm_id=%p accel=%d state=%d rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n",606 __builtin_return_address(0), cm_node, cm_node->cm_id,607 cm_node->accelerated, cm_node->state, cm_node->rem_port,608 cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr);609 610 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 0,611 1);612}613 614/**615 * irdma_active_open_err - send event for active side cm error616 * @cm_node: connection's node617 * @reset: Flag to send reset or not618 */619static void irdma_active_open_err(struct irdma_cm_node *cm_node, bool reset)620{621 trace_irdma_active_open_err(cm_node, reset,622 __builtin_return_address(0));623 irdma_cleanup_retrans_entry(cm_node);624 cm_node->cm_core->stats_connect_errs++;625 if (reset) {626 ibdev_dbg(&cm_node->iwdev->ibdev,627 "CM: cm_node=%p state=%d\n", cm_node,628 cm_node->state);629 refcount_inc(&cm_node->refcnt);630 irdma_send_reset(cm_node);631 }632 633 cm_node->state = IRDMA_CM_STATE_CLOSED;634 irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);635}636 637/**638 * irdma_passive_open_err - handle passive side cm error639 * @cm_node: connection's node640 * @reset: send reset or just free cm_node641 */642static void irdma_passive_open_err(struct irdma_cm_node *cm_node, bool reset)643{644 irdma_cleanup_retrans_entry(cm_node);645 cm_node->cm_core->stats_passive_errs++;646 cm_node->state = IRDMA_CM_STATE_CLOSED;647 ibdev_dbg(&cm_node->iwdev->ibdev, "CM: cm_node=%p state =%d\n",648 cm_node, cm_node->state);649 trace_irdma_passive_open_err(cm_node, reset,650 __builtin_return_address(0));651 if (reset)652 irdma_send_reset(cm_node);653 else654 irdma_rem_ref_cm_node(cm_node);655}656 657/**658 * irdma_event_connect_error - to create connect error event659 * @event: cm information for connect event660 */661static void irdma_event_connect_error(struct irdma_cm_event *event)662{663 struct irdma_qp *iwqp;664 struct iw_cm_id *cm_id;665 666 cm_id = event->cm_node->cm_id;667 if (!cm_id)668 return;669 670 iwqp = cm_id->provider_data;671 672 if (!iwqp || !iwqp->iwdev)673 return;674 675 iwqp->cm_id = NULL;676 cm_id->provider_data = NULL;677 irdma_send_cm_event(event->cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY,678 -ECONNRESET);679 irdma_rem_ref_cm_node(event->cm_node);680}681 682/**683 * irdma_process_options - process options from TCP header684 * @cm_node: connection's node685 * @optionsloc: point to start of options686 * @optionsize: size of all options687 * @syn_pkt: flag if syn packet688 */689static int irdma_process_options(struct irdma_cm_node *cm_node, u8 *optionsloc,690 u32 optionsize, u32 syn_pkt)691{692 u32 tmp;693 u32 offset = 0;694 union all_known_options *all_options;695 char got_mss_option = 0;696 697 while (offset < optionsize) {698 all_options = (union all_known_options *)(optionsloc + offset);699 switch (all_options->base.optionnum) {700 case OPTION_NUM_EOL:701 offset = optionsize;702 break;703 case OPTION_NUM_NONE:704 offset += 1;705 continue;706 case OPTION_NUM_MSS:707 ibdev_dbg(&cm_node->iwdev->ibdev,708 "CM: MSS Length: %d Offset: %d Size: %d\n",709 all_options->mss.len, offset, optionsize);710 got_mss_option = 1;711 if (all_options->mss.len != 4)712 return -EINVAL;713 tmp = ntohs(all_options->mss.mss);714 if ((cm_node->ipv4 &&715 (tmp + IRDMA_MTU_TO_MSS_IPV4) < IRDMA_MIN_MTU_IPV4) ||716 (!cm_node->ipv4 &&717 (tmp + IRDMA_MTU_TO_MSS_IPV6) < IRDMA_MIN_MTU_IPV6))718 return -EINVAL;719 if (tmp < cm_node->tcp_cntxt.mss)720 cm_node->tcp_cntxt.mss = tmp;721 break;722 case OPTION_NUM_WINDOW_SCALE:723 cm_node->tcp_cntxt.snd_wscale =724 all_options->windowscale.shiftcount;725 break;726 default:727 ibdev_dbg(&cm_node->iwdev->ibdev,728 "CM: Unsupported TCP Option: %x\n",729 all_options->base.optionnum);730 break;731 }732 offset += all_options->base.len;733 }734 if (!got_mss_option && syn_pkt)735 cm_node->tcp_cntxt.mss = IRDMA_CM_DEFAULT_MSS;736 737 return 0;738}739 740/**741 * irdma_handle_tcp_options - setup TCP context info after parsing TCP options742 * @cm_node: connection's node743 * @tcph: pointer tcp header744 * @optionsize: size of options rcvd745 * @passive: active or passive flag746 */747static int irdma_handle_tcp_options(struct irdma_cm_node *cm_node,748 struct tcphdr *tcph, int optionsize,749 int passive)750{751 u8 *optionsloc = (u8 *)&tcph[1];752 int ret;753 754 if (optionsize) {755 ret = irdma_process_options(cm_node, optionsloc, optionsize,756 (u32)tcph->syn);757 if (ret) {758 ibdev_dbg(&cm_node->iwdev->ibdev,759 "CM: Node %p, Sending Reset\n", cm_node);760 if (passive)761 irdma_passive_open_err(cm_node, true);762 else763 irdma_active_open_err(cm_node, true);764 return ret;765 }766 }767 768 cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->window)769 << cm_node->tcp_cntxt.snd_wscale;770 771 if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd)772 cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd;773 774 return 0;775}776 777/**778 * irdma_build_mpa_v1 - build a MPA V1 frame779 * @cm_node: connection's node780 * @start_addr: address where to build frame781 * @mpa_key: to do read0 or write0782 */783static void irdma_build_mpa_v1(struct irdma_cm_node *cm_node, void *start_addr,784 u8 mpa_key)785{786 struct ietf_mpa_v1 *mpa_frame = start_addr;787 788 switch (mpa_key) {789 case MPA_KEY_REQUEST:790 memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE);791 break;792 case MPA_KEY_REPLY:793 memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE);794 break;795 default:796 break;797 }798 mpa_frame->flags = IETF_MPA_FLAGS_CRC;799 mpa_frame->rev = cm_node->mpa_frame_rev;800 mpa_frame->priv_data_len = htons(cm_node->pdata.size);801}802 803/**804 * irdma_build_mpa_v2 - build a MPA V2 frame805 * @cm_node: connection's node806 * @start_addr: buffer start address807 * @mpa_key: to do read0 or write0808 */809static void irdma_build_mpa_v2(struct irdma_cm_node *cm_node, void *start_addr,810 u8 mpa_key)811{812 struct ietf_mpa_v2 *mpa_frame = start_addr;813 struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg;814 u16 ctrl_ird, ctrl_ord;815 816 /* initialize the upper 5 bytes of the frame */817 irdma_build_mpa_v1(cm_node, start_addr, mpa_key);818 mpa_frame->flags |= IETF_MPA_V2_FLAG;819 if (cm_node->iwdev->iw_ooo) {820 mpa_frame->flags |= IETF_MPA_FLAGS_MARKERS;821 cm_node->rcv_mark_en = true;822 }823 mpa_frame->priv_data_len = cpu_to_be16(be16_to_cpu(mpa_frame->priv_data_len) +824 IETF_RTR_MSG_SIZE);825 826 /* initialize RTR msg */827 if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) {828 ctrl_ird = IETF_NO_IRD_ORD;829 ctrl_ord = IETF_NO_IRD_ORD;830 } else {831 ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ?832 IETF_NO_IRD_ORD :833 cm_node->ird_size;834 ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ?835 IETF_NO_IRD_ORD :836 cm_node->ord_size;837 }838 ctrl_ird |= IETF_PEER_TO_PEER;839 840 switch (mpa_key) {841 case MPA_KEY_REQUEST:842 ctrl_ord |= IETF_RDMA0_WRITE;843 ctrl_ord |= IETF_RDMA0_READ;844 break;845 case MPA_KEY_REPLY:846 switch (cm_node->send_rdma0_op) {847 case SEND_RDMA_WRITE_ZERO:848 ctrl_ord |= IETF_RDMA0_WRITE;849 break;850 case SEND_RDMA_READ_ZERO:851 ctrl_ord |= IETF_RDMA0_READ;852 break;853 }854 break;855 default:856 break;857 }858 rtr_msg->ctrl_ird = htons(ctrl_ird);859 rtr_msg->ctrl_ord = htons(ctrl_ord);860}861 862/**863 * irdma_cm_build_mpa_frame - build mpa frame for mpa version 1 or version 2864 * @cm_node: connection's node865 * @mpa: mpa: data buffer866 * @mpa_key: to do read0 or write0867 */868static int irdma_cm_build_mpa_frame(struct irdma_cm_node *cm_node,869 struct irdma_kmem_info *mpa, u8 mpa_key)870{871 int hdr_len = 0;872 873 switch (cm_node->mpa_frame_rev) {874 case IETF_MPA_V1:875 hdr_len = sizeof(struct ietf_mpa_v1);876 irdma_build_mpa_v1(cm_node, mpa->addr, mpa_key);877 break;878 case IETF_MPA_V2:879 hdr_len = sizeof(struct ietf_mpa_v2);880 irdma_build_mpa_v2(cm_node, mpa->addr, mpa_key);881 break;882 default:883 break;884 }885 886 return hdr_len;887}888 889/**890 * irdma_send_mpa_request - active node send mpa request to passive node891 * @cm_node: connection's node892 */893static int irdma_send_mpa_request(struct irdma_cm_node *cm_node)894{895 struct irdma_puda_buf *sqbuf;896 897 cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame;898 cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node,899 &cm_node->mpa_hdr,900 MPA_KEY_REQUEST);901 if (!cm_node->mpa_hdr.size) {902 ibdev_dbg(&cm_node->iwdev->ibdev,903 "CM: mpa size = %d\n", cm_node->mpa_hdr.size);904 return -EINVAL;905 }906 907 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL,908 &cm_node->mpa_hdr,909 &cm_node->pdata, SET_ACK);910 if (!sqbuf)911 return -ENOMEM;912 913 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,914 0);915}916 917/**918 * irdma_send_mpa_reject -919 * @cm_node: connection's node920 * @pdata: reject data for connection921 * @plen: length of reject data922 */923static int irdma_send_mpa_reject(struct irdma_cm_node *cm_node,924 const void *pdata, u8 plen)925{926 struct irdma_puda_buf *sqbuf;927 struct irdma_mpa_priv_info priv_info;928 929 cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame;930 cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node,931 &cm_node->mpa_hdr,932 MPA_KEY_REPLY);933 934 cm_node->mpa_frame.flags |= IETF_MPA_FLAGS_REJECT;935 priv_info.addr = pdata;936 priv_info.size = plen;937 938 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL,939 &cm_node->mpa_hdr, &priv_info,940 SET_ACK | SET_FIN);941 if (!sqbuf)942 return -ENOMEM;943 944 cm_node->state = IRDMA_CM_STATE_FIN_WAIT1;945 946 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,947 0);948}949 950/**951 * irdma_negotiate_mpa_v2_ird_ord - negotiate MPAv2 IRD/ORD952 * @cm_node: connection's node953 * @buf: Data pointer954 */955static int irdma_negotiate_mpa_v2_ird_ord(struct irdma_cm_node *cm_node,956 u8 *buf)957{958 struct ietf_mpa_v2 *mpa_v2_frame;959 struct ietf_rtr_msg *rtr_msg;960 u16 ird_size;961 u16 ord_size;962 u16 ctrl_ord;963 u16 ctrl_ird;964 965 mpa_v2_frame = (struct ietf_mpa_v2 *)buf;966 rtr_msg = &mpa_v2_frame->rtr_msg;967 968 /* parse rtr message */969 ctrl_ord = ntohs(rtr_msg->ctrl_ord);970 ctrl_ird = ntohs(rtr_msg->ctrl_ird);971 ird_size = ctrl_ird & IETF_NO_IRD_ORD;972 ord_size = ctrl_ord & IETF_NO_IRD_ORD;973 974 if (!(ctrl_ird & IETF_PEER_TO_PEER))975 return -EOPNOTSUPP;976 977 if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) {978 cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD;979 goto negotiate_done;980 }981 982 if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) {983 /* responder */984 if (!ord_size && (ctrl_ord & IETF_RDMA0_READ))985 cm_node->ird_size = 1;986 if (cm_node->ord_size > ird_size)987 cm_node->ord_size = ird_size;988 } else {989 /* initiator */990 if (!ird_size && (ctrl_ord & IETF_RDMA0_READ))991 /* Remote peer doesn't support RDMA0_READ */992 return -EOPNOTSUPP;993 994 if (cm_node->ord_size > ird_size)995 cm_node->ord_size = ird_size;996 997 if (cm_node->ird_size < ord_size)998 /* no resources available */999 return -EINVAL;1000 }1001 1002negotiate_done:1003 if (ctrl_ord & IETF_RDMA0_READ)1004 cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;1005 else if (ctrl_ord & IETF_RDMA0_WRITE)1006 cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO;1007 else1008 /* Not supported RDMA0 operation */1009 return -EOPNOTSUPP;1010 1011 ibdev_dbg(&cm_node->iwdev->ibdev,1012 "CM: MPAV2 Negotiated ORD: %d, IRD: %d\n",1013 cm_node->ord_size, cm_node->ird_size);1014 trace_irdma_negotiate_mpa_v2(cm_node);1015 return 0;1016}1017 1018/**1019 * irdma_parse_mpa - process an IETF MPA frame1020 * @cm_node: connection's node1021 * @buf: Data pointer1022 * @type: to return accept or reject1023 * @len: Len of mpa buffer1024 */1025static int irdma_parse_mpa(struct irdma_cm_node *cm_node, u8 *buf, u32 *type,1026 u32 len)1027{1028 struct ietf_mpa_v1 *mpa_frame;1029 int mpa_hdr_len, priv_data_len, ret;1030 1031 *type = IRDMA_MPA_REQUEST_ACCEPT;1032 1033 if (len < sizeof(struct ietf_mpa_v1)) {1034 ibdev_dbg(&cm_node->iwdev->ibdev,1035 "CM: ietf buffer small (%x)\n", len);1036 return -EINVAL;1037 }1038 1039 mpa_frame = (struct ietf_mpa_v1 *)buf;1040 mpa_hdr_len = sizeof(struct ietf_mpa_v1);1041 priv_data_len = ntohs(mpa_frame->priv_data_len);1042 1043 if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) {1044 ibdev_dbg(&cm_node->iwdev->ibdev,1045 "CM: private_data too big %d\n", priv_data_len);1046 return -EOVERFLOW;1047 }1048 1049 if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) {1050 ibdev_dbg(&cm_node->iwdev->ibdev,1051 "CM: unsupported mpa rev = %d\n", mpa_frame->rev);1052 return -EINVAL;1053 }1054 1055 if (mpa_frame->rev > cm_node->mpa_frame_rev) {1056 ibdev_dbg(&cm_node->iwdev->ibdev, "CM: rev %d\n",1057 mpa_frame->rev);1058 return -EINVAL;1059 }1060 1061 cm_node->mpa_frame_rev = mpa_frame->rev;1062 if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) {1063 if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ,1064 IETF_MPA_KEY_SIZE)) {1065 ibdev_dbg(&cm_node->iwdev->ibdev,1066 "CM: Unexpected MPA Key received\n");1067 return -EINVAL;1068 }1069 } else {1070 if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP,1071 IETF_MPA_KEY_SIZE)) {1072 ibdev_dbg(&cm_node->iwdev->ibdev,1073 "CM: Unexpected MPA Key received\n");1074 return -EINVAL;1075 }1076 }1077 1078 if (priv_data_len + mpa_hdr_len > len) {1079 ibdev_dbg(&cm_node->iwdev->ibdev,1080 "CM: ietf buffer len(%x + %x != %x)\n",1081 priv_data_len, mpa_hdr_len, len);1082 return -EOVERFLOW;1083 }1084 1085 if (len > IRDMA_MAX_CM_BUF) {1086 ibdev_dbg(&cm_node->iwdev->ibdev,1087 "CM: ietf buffer large len = %d\n", len);1088 return -EOVERFLOW;1089 }1090 1091 switch (mpa_frame->rev) {1092 case IETF_MPA_V2:1093 mpa_hdr_len += IETF_RTR_MSG_SIZE;1094 ret = irdma_negotiate_mpa_v2_ird_ord(cm_node, buf);1095 if (ret)1096 return ret;1097 break;1098 case IETF_MPA_V1:1099 default:1100 break;1101 }1102 1103 memcpy(cm_node->pdata_buf, buf + mpa_hdr_len, priv_data_len);1104 cm_node->pdata.size = priv_data_len;1105 1106 if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT)1107 *type = IRDMA_MPA_REQUEST_REJECT;1108 1109 if (mpa_frame->flags & IETF_MPA_FLAGS_MARKERS)1110 cm_node->snd_mark_en = true;1111 1112 return 0;1113}1114 1115/**1116 * irdma_schedule_cm_timer1117 * @cm_node: connection's node1118 * @sqbuf: buffer to send1119 * @type: if it is send or close1120 * @send_retrans: if rexmits to be done1121 * @close_when_complete: is cm_node to be removed1122 *1123 * note - cm_node needs to be protected before calling this. Encase in:1124 * irdma_rem_ref_cm_node(cm_core, cm_node);1125 * irdma_schedule_cm_timer(...)1126 * refcount_inc(&cm_node->refcnt);1127 */1128int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node,1129 struct irdma_puda_buf *sqbuf,1130 enum irdma_timer_type type, int send_retrans,1131 int close_when_complete)1132{1133 struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;1134 struct irdma_cm_core *cm_core = cm_node->cm_core;1135 struct irdma_timer_entry *new_send;1136 u32 was_timer_set;1137 unsigned long flags;1138 1139 new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC);1140 if (!new_send) {1141 if (type != IRDMA_TIMER_TYPE_CLOSE)1142 irdma_free_sqbuf(vsi, sqbuf);1143 return -ENOMEM;1144 }1145 1146 new_send->retrycount = IRDMA_DEFAULT_RETRYS;1147 new_send->retranscount = IRDMA_DEFAULT_RETRANS;1148 new_send->sqbuf = sqbuf;1149 new_send->timetosend = jiffies;1150 new_send->type = type;1151 new_send->send_retrans = send_retrans;1152 new_send->close_when_complete = close_when_complete;1153 1154 if (type == IRDMA_TIMER_TYPE_CLOSE) {1155 new_send->timetosend += (HZ / 10);1156 if (cm_node->close_entry) {1157 kfree(new_send);1158 ibdev_dbg(&cm_node->iwdev->ibdev,1159 "CM: already close entry\n");1160 return -EINVAL;1161 }1162 1163 cm_node->close_entry = new_send;1164 } else { /* type == IRDMA_TIMER_TYPE_SEND */1165 spin_lock_irqsave(&cm_node->retrans_list_lock, flags);1166 cm_node->send_entry = new_send;1167 refcount_inc(&cm_node->refcnt);1168 spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);1169 new_send->timetosend = jiffies + IRDMA_RETRY_TIMEOUT;1170 1171 refcount_inc(&sqbuf->refcount);1172 irdma_puda_send_buf(vsi->ilq, sqbuf);1173 if (!send_retrans) {1174 irdma_cleanup_retrans_entry(cm_node);1175 if (close_when_complete)1176 irdma_rem_ref_cm_node(cm_node);1177 return 0;1178 }1179 }1180 1181 spin_lock_irqsave(&cm_core->ht_lock, flags);1182 was_timer_set = timer_pending(&cm_core->tcp_timer);1183 1184 if (!was_timer_set) {1185 cm_core->tcp_timer.expires = new_send->timetosend;1186 add_timer(&cm_core->tcp_timer);1187 }1188 spin_unlock_irqrestore(&cm_core->ht_lock, flags);1189 1190 return 0;1191}1192 1193/**1194 * irdma_retrans_expired - Could not rexmit the packet1195 * @cm_node: connection's node1196 */1197static void irdma_retrans_expired(struct irdma_cm_node *cm_node)1198{1199 enum irdma_cm_node_state state = cm_node->state;1200 1201 cm_node->state = IRDMA_CM_STATE_CLOSED;1202 switch (state) {1203 case IRDMA_CM_STATE_SYN_RCVD:1204 case IRDMA_CM_STATE_CLOSING:1205 irdma_rem_ref_cm_node(cm_node);1206 break;1207 case IRDMA_CM_STATE_FIN_WAIT1:1208 case IRDMA_CM_STATE_LAST_ACK:1209 irdma_send_reset(cm_node);1210 break;1211 default:1212 refcount_inc(&cm_node->refcnt);1213 irdma_send_reset(cm_node);1214 irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);1215 break;1216 }1217}1218 1219/**1220 * irdma_handle_close_entry - for handling retry/timeouts1221 * @cm_node: connection's node1222 * @rem_node: flag for remove cm_node1223 */1224static void irdma_handle_close_entry(struct irdma_cm_node *cm_node,1225 u32 rem_node)1226{1227 struct irdma_timer_entry *close_entry = cm_node->close_entry;1228 struct irdma_qp *iwqp;1229 unsigned long flags;1230 1231 if (!close_entry)1232 return;1233 iwqp = (struct irdma_qp *)close_entry->sqbuf;1234 if (iwqp) {1235 spin_lock_irqsave(&iwqp->lock, flags);1236 if (iwqp->cm_id) {1237 iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED;1238 iwqp->hw_iwarp_state = IRDMA_QP_STATE_ERROR;1239 iwqp->last_aeq = IRDMA_AE_RESET_SENT;1240 iwqp->ibqp_state = IB_QPS_ERR;1241 spin_unlock_irqrestore(&iwqp->lock, flags);1242 irdma_cm_disconn(iwqp);1243 } else {1244 spin_unlock_irqrestore(&iwqp->lock, flags);1245 }1246 } else if (rem_node) {1247 /* TIME_WAIT state */1248 irdma_rem_ref_cm_node(cm_node);1249 }1250 1251 kfree(close_entry);1252 cm_node->close_entry = NULL;1253}1254 1255/**1256 * irdma_cm_timer_tick - system's timer expired callback1257 * @t: Pointer to timer_list1258 */1259static void irdma_cm_timer_tick(struct timer_list *t)1260{1261 unsigned long nexttimeout = jiffies + IRDMA_LONG_TIME;1262 struct irdma_cm_node *cm_node;1263 struct irdma_timer_entry *send_entry, *close_entry;1264 struct list_head *list_core_temp;1265 struct list_head *list_node;1266 struct irdma_cm_core *cm_core = from_timer(cm_core, t, tcp_timer);1267 struct irdma_sc_vsi *vsi;1268 u32 settimer = 0;1269 unsigned long timetosend;1270 unsigned long flags;1271 struct list_head timer_list;1272 1273 INIT_LIST_HEAD(&timer_list);1274 1275 rcu_read_lock();1276 irdma_timer_list_prep(cm_core, &timer_list);1277 rcu_read_unlock();1278 1279 list_for_each_safe (list_node, list_core_temp, &timer_list) {1280 cm_node = container_of(list_node, struct irdma_cm_node,1281 timer_entry);1282 close_entry = cm_node->close_entry;1283 1284 if (close_entry) {1285 if (time_after(close_entry->timetosend, jiffies)) {1286 if (nexttimeout > close_entry->timetosend ||1287 !settimer) {1288 nexttimeout = close_entry->timetosend;1289 settimer = 1;1290 }1291 } else {1292 irdma_handle_close_entry(cm_node, 1);1293 }1294 }1295 1296 spin_lock_irqsave(&cm_node->retrans_list_lock, flags);1297 1298 send_entry = cm_node->send_entry;1299 if (!send_entry)1300 goto done;1301 if (time_after(send_entry->timetosend, jiffies)) {1302 if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) {1303 if (nexttimeout > send_entry->timetosend ||1304 !settimer) {1305 nexttimeout = send_entry->timetosend;1306 settimer = 1;1307 }1308 } else {1309 irdma_free_retrans_entry(cm_node);1310 }1311 goto done;1312 }1313 1314 if (cm_node->state == IRDMA_CM_STATE_OFFLOADED ||1315 cm_node->state == IRDMA_CM_STATE_CLOSED) {1316 irdma_free_retrans_entry(cm_node);1317 goto done;1318 }1319 1320 if (!send_entry->retranscount || !send_entry->retrycount) {1321 irdma_free_retrans_entry(cm_node);1322 1323 spin_unlock_irqrestore(&cm_node->retrans_list_lock,1324 flags);1325 irdma_retrans_expired(cm_node);1326 cm_node->state = IRDMA_CM_STATE_CLOSED;1327 spin_lock_irqsave(&cm_node->retrans_list_lock, flags);1328 goto done;1329 }1330 spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);1331 1332 vsi = &cm_node->iwdev->vsi;1333 if (!cm_node->ack_rcvd) {1334 refcount_inc(&send_entry->sqbuf->refcount);1335 irdma_puda_send_buf(vsi->ilq, send_entry->sqbuf);1336 cm_node->cm_core->stats_pkt_retrans++;1337 }1338 1339 spin_lock_irqsave(&cm_node->retrans_list_lock, flags);1340 if (send_entry->send_retrans) {1341 send_entry->retranscount--;1342 timetosend = (IRDMA_RETRY_TIMEOUT <<1343 (IRDMA_DEFAULT_RETRANS -1344 send_entry->retranscount));1345 1346 send_entry->timetosend = jiffies +1347 min(timetosend, IRDMA_MAX_TIMEOUT);1348 if (nexttimeout > send_entry->timetosend || !settimer) {1349 nexttimeout = send_entry->timetosend;1350 settimer = 1;1351 }1352 } else {1353 int close_when_complete;1354 1355 close_when_complete = send_entry->close_when_complete;1356 irdma_free_retrans_entry(cm_node);1357 if (close_when_complete)1358 irdma_rem_ref_cm_node(cm_node);1359 }1360done:1361 spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);1362 irdma_rem_ref_cm_node(cm_node);1363 }1364 1365 if (settimer) {1366 spin_lock_irqsave(&cm_core->ht_lock, flags);1367 if (!timer_pending(&cm_core->tcp_timer)) {1368 cm_core->tcp_timer.expires = nexttimeout;1369 add_timer(&cm_core->tcp_timer);1370 }1371 spin_unlock_irqrestore(&cm_core->ht_lock, flags);1372 }1373}1374 1375/**1376 * irdma_send_syn - send SYN packet1377 * @cm_node: connection's node1378 * @sendack: flag to set ACK bit or not1379 */1380int irdma_send_syn(struct irdma_cm_node *cm_node, u32 sendack)1381{1382 struct irdma_puda_buf *sqbuf;1383 int flags = SET_SYN;1384 char optionsbuf[sizeof(struct option_mss) +1385 sizeof(struct option_windowscale) +1386 sizeof(struct option_base) + TCP_OPTIONS_PADDING];1387 struct irdma_kmem_info opts;1388 int optionssize = 0;1389 /* Sending MSS option */1390 union all_known_options *options;1391 1392 opts.addr = optionsbuf;1393 if (!cm_node)1394 return -EINVAL;1395 1396 options = (union all_known_options *)&optionsbuf[optionssize];1397 options->mss.optionnum = OPTION_NUM_MSS;1398 options->mss.len = sizeof(struct option_mss);1399 options->mss.mss = htons(cm_node->tcp_cntxt.mss);1400 optionssize += sizeof(struct option_mss);1401 1402 options = (union all_known_options *)&optionsbuf[optionssize];1403 options->windowscale.optionnum = OPTION_NUM_WINDOW_SCALE;1404 options->windowscale.len = sizeof(struct option_windowscale);1405 options->windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale;1406 optionssize += sizeof(struct option_windowscale);1407 options = (union all_known_options *)&optionsbuf[optionssize];1408 options->eol = OPTION_NUM_EOL;1409 optionssize += 1;1410 1411 if (sendack)1412 flags |= SET_ACK;1413 1414 opts.size = optionssize;1415 1416 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, &opts, NULL, NULL,1417 flags);1418 if (!sqbuf)1419 return -ENOMEM;1420 1421 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,1422 0);1423}1424 1425/**1426 * irdma_send_ack - Send ACK packet1427 * @cm_node: connection's node1428 */1429void irdma_send_ack(struct irdma_cm_node *cm_node)1430{1431 struct irdma_puda_buf *sqbuf;1432 struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;1433 1434 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL,1435 SET_ACK);1436 if (sqbuf)1437 irdma_puda_send_buf(vsi->ilq, sqbuf);1438}1439 1440/**1441 * irdma_send_fin - Send FIN pkt1442 * @cm_node: connection's node1443 */1444static int irdma_send_fin(struct irdma_cm_node *cm_node)1445{1446 struct irdma_puda_buf *sqbuf;1447 1448 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL,1449 SET_ACK | SET_FIN);1450 if (!sqbuf)1451 return -ENOMEM;1452 1453 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,1454 0);1455}1456 1457/**1458 * irdma_find_listener - find a cm node listening on this addr-port pair1459 * @cm_core: cm's core1460 * @dst_addr: listener ip addr1461 * @ipv4: flag indicating IPv4 when true1462 * @dst_port: listener tcp port num1463 * @vlan_id: virtual LAN ID1464 * @listener_state: state to match with listen node's1465 */1466static struct irdma_cm_listener *1467irdma_find_listener(struct irdma_cm_core *cm_core, u32 *dst_addr, bool ipv4,1468 u16 dst_port, u16 vlan_id,1469 enum irdma_cm_listener_state listener_state)1470{1471 struct irdma_cm_listener *listen_node;1472 static const u32 ip_zero[4] = { 0, 0, 0, 0 };1473 u32 listen_addr[4];1474 u16 listen_port;1475 unsigned long flags;1476 1477 /* walk list and find cm_node associated with this session ID */1478 spin_lock_irqsave(&cm_core->listen_list_lock, flags);1479 list_for_each_entry (listen_node, &cm_core->listen_list, list) {1480 memcpy(listen_addr, listen_node->loc_addr, sizeof(listen_addr));1481 listen_port = listen_node->loc_port;1482 if (listen_node->ipv4 != ipv4 || listen_port != dst_port ||1483 !(listener_state & listen_node->listener_state))1484 continue;1485 /* compare node pair, return node handle if a match */1486 if (!memcmp(listen_addr, ip_zero, sizeof(listen_addr)) ||1487 (!memcmp(listen_addr, dst_addr, sizeof(listen_addr)) &&1488 vlan_id == listen_node->vlan_id)) {1489 refcount_inc(&listen_node->refcnt);1490 spin_unlock_irqrestore(&cm_core->listen_list_lock,1491 flags);1492 trace_irdma_find_listener(listen_node);1493 return listen_node;1494 }1495 }1496 spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);1497 1498 return NULL;1499}1500 1501/**1502 * irdma_del_multiple_qhash - Remove qhash and child listens1503 * @iwdev: iWarp device1504 * @cm_info: CM info for parent listen node1505 * @cm_parent_listen_node: The parent listen node1506 */1507static int irdma_del_multiple_qhash(struct irdma_device *iwdev,1508 struct irdma_cm_info *cm_info,1509 struct irdma_cm_listener *cm_parent_listen_node)1510{1511 struct irdma_cm_listener *child_listen_node;1512 struct list_head *pos, *tpos;1513 unsigned long flags;1514 int ret = -EINVAL;1515 1516 spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);1517 list_for_each_safe (pos, tpos,1518 &cm_parent_listen_node->child_listen_list) {1519 child_listen_node = list_entry(pos, struct irdma_cm_listener,1520 child_listen_list);1521 if (child_listen_node->ipv4)1522 ibdev_dbg(&iwdev->ibdev,1523 "CM: removing child listen for IP=%pI4, port=%d, vlan=%d\n",1524 child_listen_node->loc_addr,1525 child_listen_node->loc_port,1526 child_listen_node->vlan_id);1527 else1528 ibdev_dbg(&iwdev->ibdev,1529 "CM: removing child listen for IP=%pI6, port=%d, vlan=%d\n",1530 child_listen_node->loc_addr,1531 child_listen_node->loc_port,1532 child_listen_node->vlan_id);1533 trace_irdma_del_multiple_qhash(child_listen_node);1534 list_del(pos);1535 memcpy(cm_info->loc_addr, child_listen_node->loc_addr,1536 sizeof(cm_info->loc_addr));1537 cm_info->vlan_id = child_listen_node->vlan_id;1538 if (child_listen_node->qhash_set) {1539 ret = irdma_manage_qhash(iwdev, cm_info,1540 IRDMA_QHASH_TYPE_TCP_SYN,1541 IRDMA_QHASH_MANAGE_TYPE_DELETE,1542 NULL, false);1543 child_listen_node->qhash_set = false;1544 } else {1545 ret = 0;1546 }1547 ibdev_dbg(&iwdev->ibdev,1548 "CM: Child listen node freed = %p\n",1549 child_listen_node);1550 kfree(child_listen_node);1551 cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++;1552 }1553 spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);1554 1555 return ret;1556}1557 1558static u8 irdma_iw_get_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4)1559{1560 struct net_device *ndev = NULL;1561 1562 rcu_read_lock();1563 if (ipv4) {1564 ndev = ip_dev_find(&init_net, htonl(loc_addr[0]));1565 } else if (IS_ENABLED(CONFIG_IPV6)) {1566 struct net_device *ip_dev;1567 struct in6_addr laddr6;1568 1569 irdma_copy_ip_htonl(laddr6.in6_u.u6_addr32, loc_addr);1570 1571 for_each_netdev_rcu (&init_net, ip_dev) {1572 if (ipv6_chk_addr(&init_net, &laddr6, ip_dev, 1)) {1573 ndev = ip_dev;1574 break;1575 }1576 }1577 }1578 1579 if (!ndev)1580 goto done;1581 if (is_vlan_dev(ndev))1582 prio = (vlan_dev_get_egress_qos_mask(ndev, prio) & VLAN_PRIO_MASK)1583 >> VLAN_PRIO_SHIFT;1584 if (ipv4)1585 dev_put(ndev);1586 1587done:1588 rcu_read_unlock();1589 1590 return prio;1591}1592 1593/**1594 * irdma_get_vlan_mac_ipv6 - Gets the vlan and mac1595 * @addr: local IPv6 address1596 * @vlan_id: vlan id for the given IPv6 address1597 * @mac: mac address for the given IPv6 address1598 *1599 * Returns the vlan id and mac for an IPv6 address.1600 */1601void irdma_get_vlan_mac_ipv6(u32 *addr, u16 *vlan_id, u8 *mac)1602{1603 struct net_device *ip_dev = NULL;1604 struct in6_addr laddr6;1605 1606 if (!IS_ENABLED(CONFIG_IPV6))1607 return;1608 1609 irdma_copy_ip_htonl(laddr6.in6_u.u6_addr32, addr);1610 if (vlan_id)1611 *vlan_id = 0xFFFF; /* Match rdma_vlan_dev_vlan_id() */1612 if (mac)1613 eth_zero_addr(mac);1614 1615 rcu_read_lock();1616 for_each_netdev_rcu (&init_net, ip_dev) {1617 if (ipv6_chk_addr(&init_net, &laddr6, ip_dev, 1)) {1618 if (vlan_id)1619 *vlan_id = rdma_vlan_dev_vlan_id(ip_dev);1620 if (ip_dev->dev_addr && mac)1621 ether_addr_copy(mac, ip_dev->dev_addr);1622 break;1623 }1624 }1625 rcu_read_unlock();1626}1627 1628/**1629 * irdma_get_vlan_ipv4 - Returns the vlan_id for IPv4 address1630 * @addr: local IPv4 address1631 */1632u16 irdma_get_vlan_ipv4(u32 *addr)1633{1634 struct net_device *netdev;1635 u16 vlan_id = 0xFFFF;1636 1637 netdev = ip_dev_find(&init_net, htonl(addr[0]));1638 if (netdev) {1639 vlan_id = rdma_vlan_dev_vlan_id(netdev);1640 dev_put(netdev);1641 }1642 1643 return vlan_id;1644}1645 1646/**1647 * irdma_add_mqh_6 - Adds multiple qhashes for IPv61648 * @iwdev: iWarp device1649 * @cm_info: CM info for parent listen node1650 * @cm_parent_listen_node: The parent listen node1651 *1652 * Adds a qhash and a child listen node for every IPv6 address1653 * on the adapter and adds the associated qhash filter1654 */1655static int irdma_add_mqh_6(struct irdma_device *iwdev,1656 struct irdma_cm_info *cm_info,1657 struct irdma_cm_listener *cm_parent_listen_node)1658{1659 struct net_device *ip_dev;1660 struct inet6_dev *idev;1661 struct inet6_ifaddr *ifp, *tmp;1662 struct irdma_cm_listener *child_listen_node;1663 unsigned long flags;1664 int ret = 0;1665 1666 rtnl_lock();1667 for_each_netdev(&init_net, ip_dev) {1668 if (!(ip_dev->flags & IFF_UP))1669 continue;1670 1671 if (((rdma_vlan_dev_vlan_id(ip_dev) >= VLAN_N_VID) ||1672 (rdma_vlan_dev_real_dev(ip_dev) != iwdev->netdev)) &&1673 ip_dev != iwdev->netdev)1674 continue;1675 1676 idev = __in6_dev_get(ip_dev);1677 if (!idev) {1678 ibdev_dbg(&iwdev->ibdev, "CM: idev == NULL\n");1679 break;1680 }1681 list_for_each_entry_safe (ifp, tmp, &idev->addr_list, if_list) {1682 ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI6, vlan_id=%d, MAC=%pM\n",1683 &ifp->addr, rdma_vlan_dev_vlan_id(ip_dev),1684 ip_dev->dev_addr);1685 child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);1686 ibdev_dbg(&iwdev->ibdev, "CM: Allocating child listener %p\n",1687 child_listen_node);1688 if (!child_listen_node) {1689 ibdev_dbg(&iwdev->ibdev, "CM: listener memory allocation\n");1690 ret = -ENOMEM;1691 goto exit;1692 }1693 1694 cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev);1695 cm_parent_listen_node->vlan_id = cm_info->vlan_id;1696 memcpy(child_listen_node, cm_parent_listen_node,1697 sizeof(*child_listen_node));1698 irdma_copy_ip_ntohl(child_listen_node->loc_addr,1699 ifp->addr.in6_u.u6_addr32);1700 memcpy(cm_info->loc_addr, child_listen_node->loc_addr,1701 sizeof(cm_info->loc_addr));1702 if (!iwdev->vsi.dscp_mode)1703 cm_info->user_pri =1704 irdma_iw_get_vlan_prio(child_listen_node->loc_addr,1705 cm_info->user_pri,1706 false);1707 1708 ret = irdma_manage_qhash(iwdev, cm_info,1709 IRDMA_QHASH_TYPE_TCP_SYN,1710 IRDMA_QHASH_MANAGE_TYPE_ADD,1711 NULL, true);1712 if (ret) {1713 kfree(child_listen_node);1714 continue;1715 }1716 1717 trace_irdma_add_mqh_6(iwdev, child_listen_node,1718 ip_dev->dev_addr);1719 1720 child_listen_node->qhash_set = true;1721 spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);1722 list_add(&child_listen_node->child_listen_list,1723 &cm_parent_listen_node->child_listen_list);1724 spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);1725 cm_parent_listen_node->cm_core->stats_listen_nodes_created++;1726 }1727 }1728exit:1729 rtnl_unlock();1730 1731 return ret;1732}1733 1734/**1735 * irdma_add_mqh_4 - Adds multiple qhashes for IPv41736 * @iwdev: iWarp device1737 * @cm_info: CM info for parent listen node1738 * @cm_parent_listen_node: The parent listen node1739 *1740 * Adds a qhash and a child listen node for every IPv4 address1741 * on the adapter and adds the associated qhash filter1742 */1743static int irdma_add_mqh_4(struct irdma_device *iwdev,1744 struct irdma_cm_info *cm_info,1745 struct irdma_cm_listener *cm_parent_listen_node)1746{1747 struct net_device *ip_dev;1748 struct in_device *idev;1749 struct irdma_cm_listener *child_listen_node;1750 unsigned long flags;1751 const struct in_ifaddr *ifa;1752 int ret = 0;1753 1754 rtnl_lock();1755 for_each_netdev(&init_net, ip_dev) {1756 if (!(ip_dev->flags & IFF_UP))1757 continue;1758 1759 if (((rdma_vlan_dev_vlan_id(ip_dev) >= VLAN_N_VID) ||1760 (rdma_vlan_dev_real_dev(ip_dev) != iwdev->netdev)) &&1761 ip_dev != iwdev->netdev)1762 continue;1763 1764 idev = in_dev_get(ip_dev);1765 if (!idev)1766 continue;1767 1768 in_dev_for_each_ifa_rtnl(ifa, idev) {1769 ibdev_dbg(&iwdev->ibdev,1770 "CM: Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n",1771 &ifa->ifa_address, rdma_vlan_dev_vlan_id(ip_dev),1772 ip_dev->dev_addr);1773 child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);1774 cm_parent_listen_node->cm_core->stats_listen_nodes_created++;1775 ibdev_dbg(&iwdev->ibdev, "CM: Allocating child listener %p\n",1776 child_listen_node);1777 if (!child_listen_node) {1778 ibdev_dbg(&iwdev->ibdev, "CM: listener memory allocation\n");1779 in_dev_put(idev);1780 ret = -ENOMEM;1781 goto exit;1782 }1783 1784 cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev);1785 cm_parent_listen_node->vlan_id = cm_info->vlan_id;1786 memcpy(child_listen_node, cm_parent_listen_node,1787 sizeof(*child_listen_node));1788 child_listen_node->loc_addr[0] =1789 ntohl(ifa->ifa_address);1790 memcpy(cm_info->loc_addr, child_listen_node->loc_addr,1791 sizeof(cm_info->loc_addr));1792 if (!iwdev->vsi.dscp_mode)1793 cm_info->user_pri =1794 irdma_iw_get_vlan_prio(child_listen_node->loc_addr,1795 cm_info->user_pri,1796 true);1797 ret = irdma_manage_qhash(iwdev, cm_info,1798 IRDMA_QHASH_TYPE_TCP_SYN,1799 IRDMA_QHASH_MANAGE_TYPE_ADD,1800 NULL, true);1801 if (ret) {1802 kfree(child_listen_node);1803 cm_parent_listen_node->cm_core1804 ->stats_listen_nodes_created--;1805 continue;1806 }1807 1808 trace_irdma_add_mqh_4(iwdev, child_listen_node,1809 ip_dev->dev_addr);1810 1811 child_listen_node->qhash_set = true;1812 spin_lock_irqsave(&iwdev->cm_core.listen_list_lock,1813 flags);1814 list_add(&child_listen_node->child_listen_list,1815 &cm_parent_listen_node->child_listen_list);1816 spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);1817 }1818 in_dev_put(idev);1819 }1820exit:1821 rtnl_unlock();1822 1823 return ret;1824}1825 1826/**1827 * irdma_add_mqh - Adds multiple qhashes1828 * @iwdev: iWarp device1829 * @cm_info: CM info for parent listen node1830 * @cm_listen_node: The parent listen node1831 */1832static int irdma_add_mqh(struct irdma_device *iwdev,1833 struct irdma_cm_info *cm_info,1834 struct irdma_cm_listener *cm_listen_node)1835{1836 if (cm_info->ipv4)1837 return irdma_add_mqh_4(iwdev, cm_info, cm_listen_node);1838 else1839 return irdma_add_mqh_6(iwdev, cm_info, cm_listen_node);1840}1841 1842/**1843 * irdma_reset_list_prep - add connection nodes slated for reset to list1844 * @cm_core: cm's core1845 * @listener: pointer to listener node1846 * @reset_list: a list to which cm_node will be selected1847 */1848static void irdma_reset_list_prep(struct irdma_cm_core *cm_core,1849 struct irdma_cm_listener *listener,1850 struct list_head *reset_list)1851{1852 struct irdma_cm_node *cm_node;1853 int bkt;1854 1855 hash_for_each_rcu(cm_core->cm_hash_tbl, bkt, cm_node, list) {1856 if (cm_node->listener == listener &&1857 !cm_node->accelerated &&1858 refcount_inc_not_zero(&cm_node->refcnt))1859 list_add(&cm_node->reset_entry, reset_list);1860 }1861}1862 1863/**1864 * irdma_dec_refcnt_listen - delete listener and associated cm nodes1865 * @cm_core: cm's core1866 * @listener: pointer to listener node1867 * @free_hanging_nodes: to free associated cm_nodes1868 * @apbvt_del: flag to delete the apbvt1869 */1870static int irdma_dec_refcnt_listen(struct irdma_cm_core *cm_core,1871 struct irdma_cm_listener *listener,1872 int free_hanging_nodes, bool apbvt_del)1873{1874 int err;1875 struct list_head *list_pos;1876 struct list_head *list_temp;1877 struct irdma_cm_node *cm_node;1878 struct list_head reset_list;1879 struct irdma_cm_info nfo;1880 enum irdma_cm_node_state old_state;1881 unsigned long flags;1882 1883 trace_irdma_dec_refcnt_listen(listener, __builtin_return_address(0));1884 /* free non-accelerated child nodes for this listener */1885 INIT_LIST_HEAD(&reset_list);1886 if (free_hanging_nodes) {1887 rcu_read_lock();1888 irdma_reset_list_prep(cm_core, listener, &reset_list);1889 rcu_read_unlock();1890 }1891 1892 list_for_each_safe (list_pos, list_temp, &reset_list) {1893 cm_node = container_of(list_pos, struct irdma_cm_node,1894 reset_entry);1895 if (cm_node->state >= IRDMA_CM_STATE_FIN_WAIT1) {1896 irdma_rem_ref_cm_node(cm_node);1897 continue;1898 }1899 1900 irdma_cleanup_retrans_entry(cm_node);1901 err = irdma_send_reset(cm_node);1902 if (err) {1903 cm_node->state = IRDMA_CM_STATE_CLOSED;1904 ibdev_dbg(&cm_node->iwdev->ibdev,1905 "CM: send reset failed\n");1906 } else {1907 old_state = cm_node->state;1908 cm_node->state = IRDMA_CM_STATE_LISTENER_DESTROYED;1909 if (old_state != IRDMA_CM_STATE_MPAREQ_RCVD)1910 irdma_rem_ref_cm_node(cm_node);1911 }1912 }1913 1914 if (refcount_dec_and_test(&listener->refcnt)) {1915 spin_lock_irqsave(&cm_core->listen_list_lock, flags);1916 list_del(&listener->list);1917 spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);1918 1919 if (apbvt_del)1920 irdma_del_apbvt(listener->iwdev,1921 listener->apbvt_entry);1922 memcpy(nfo.loc_addr, listener->loc_addr, sizeof(nfo.loc_addr));1923 nfo.loc_port = listener->loc_port;1924 nfo.ipv4 = listener->ipv4;1925 nfo.vlan_id = listener->vlan_id;1926 nfo.user_pri = listener->user_pri;1927 nfo.qh_qpid = listener->iwdev->vsi.ilq->qp_id;1928 1929 if (!list_empty(&listener->child_listen_list)) {1930 irdma_del_multiple_qhash(listener->iwdev, &nfo,1931 listener);1932 } else {1933 if (listener->qhash_set)1934 irdma_manage_qhash(listener->iwdev,1935 &nfo,1936 IRDMA_QHASH_TYPE_TCP_SYN,1937 IRDMA_QHASH_MANAGE_TYPE_DELETE,1938 NULL, false);1939 }1940 1941 cm_core->stats_listen_destroyed++;1942 cm_core->stats_listen_nodes_destroyed++;1943 ibdev_dbg(&listener->iwdev->ibdev,1944 "CM: loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n",1945 listener->loc_port, listener->loc_addr, listener,1946 listener->cm_id, listener->qhash_set,1947 listener->vlan_id, apbvt_del);1948 kfree(listener);1949 listener = NULL;1950 return 0;1951 }1952 1953 return -EINVAL;1954}1955 1956/**1957 * irdma_cm_del_listen - delete a listener1958 * @cm_core: cm's core1959 * @listener: passive connection's listener1960 * @apbvt_del: flag to delete apbvt1961 */1962static int irdma_cm_del_listen(struct irdma_cm_core *cm_core,1963 struct irdma_cm_listener *listener,1964 bool apbvt_del)1965{1966 listener->listener_state = IRDMA_CM_LISTENER_PASSIVE_STATE;1967 listener->cm_id = NULL;1968 1969 return irdma_dec_refcnt_listen(cm_core, listener, 1, apbvt_del);1970}1971 1972/**1973 * irdma_addr_resolve_neigh - resolve neighbor address1974 * @iwdev: iwarp device structure1975 * @src_ip: local ip address1976 * @dst_ip: remote ip address1977 * @arpindex: if there is an arp entry1978 */1979static int irdma_addr_resolve_neigh(struct irdma_device *iwdev, u32 src_ip,1980 u32 dst_ip, int arpindex)1981{1982 struct rtable *rt;1983 struct neighbour *neigh;1984 int rc = arpindex;1985 __be32 dst_ipaddr = htonl(dst_ip);1986 __be32 src_ipaddr = htonl(src_ip);1987 1988 rt = ip_route_output(&init_net, dst_ipaddr, src_ipaddr, 0, 0,1989 RT_SCOPE_UNIVERSE);1990 if (IS_ERR(rt)) {1991 ibdev_dbg(&iwdev->ibdev, "CM: ip_route_output fail\n");1992 return -EINVAL;1993 }1994 1995 neigh = dst_neigh_lookup(&rt->dst, &dst_ipaddr);1996 if (!neigh)1997 goto exit;1998 1999 if (neigh->nud_state & NUD_VALID)2000 rc = irdma_add_arp(iwdev->rf, &dst_ip, true, neigh->ha);2001 else2002 neigh_event_send(neigh, NULL);2003 if (neigh)2004 neigh_release(neigh);2005exit:2006 ip_rt_put(rt);2007 2008 return rc;2009}2010 2011/**2012 * irdma_get_dst_ipv6 - get destination cache entry via ipv6 lookup2013 * @src_addr: local ipv6 sock address2014 * @dst_addr: destination ipv6 sock address2015 */2016static struct dst_entry *irdma_get_dst_ipv6(struct sockaddr_in6 *src_addr,2017 struct sockaddr_in6 *dst_addr)2018{2019 struct dst_entry *dst = NULL;2020 2021 if ((IS_ENABLED(CONFIG_IPV6))) {2022 struct flowi6 fl6 = {};2023 2024 fl6.daddr = dst_addr->sin6_addr;2025 fl6.saddr = src_addr->sin6_addr;2026 if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL)2027 fl6.flowi6_oif = dst_addr->sin6_scope_id;2028 2029 dst = ip6_route_output(&init_net, NULL, &fl6);2030 }2031 2032 return dst;2033}2034 2035/**2036 * irdma_addr_resolve_neigh_ipv6 - resolve neighbor ipv6 address2037 * @iwdev: iwarp device structure2038 * @src: local ip address2039 * @dest: remote ip address2040 * @arpindex: if there is an arp entry2041 */2042static int irdma_addr_resolve_neigh_ipv6(struct irdma_device *iwdev, u32 *src,2043 u32 *dest, int arpindex)2044{2045 struct neighbour *neigh;2046 int rc = arpindex;2047 struct dst_entry *dst;2048 struct sockaddr_in6 dst_addr = {};2049 struct sockaddr_in6 src_addr = {};2050 2051 dst_addr.sin6_family = AF_INET6;2052 irdma_copy_ip_htonl(dst_addr.sin6_addr.in6_u.u6_addr32, dest);2053 src_addr.sin6_family = AF_INET6;2054 irdma_copy_ip_htonl(src_addr.sin6_addr.in6_u.u6_addr32, src);2055 dst = irdma_get_dst_ipv6(&src_addr, &dst_addr);2056 if (!dst || dst->error) {2057 if (dst) {2058 dst_release(dst);2059 ibdev_dbg(&iwdev->ibdev,2060 "CM: ip6_route_output returned dst->error = %d\n",2061 dst->error);2062 }2063 return -EINVAL;2064 }2065 2066 neigh = dst_neigh_lookup(dst, dst_addr.sin6_addr.in6_u.u6_addr32);2067 if (!neigh)2068 goto exit;2069 2070 ibdev_dbg(&iwdev->ibdev, "CM: dst_neigh_lookup MAC=%pM\n",2071 neigh->ha);2072 2073 trace_irdma_addr_resolve(iwdev, neigh->ha);2074 2075 if (neigh->nud_state & NUD_VALID)2076 rc = irdma_add_arp(iwdev->rf, dest, false, neigh->ha);2077 else2078 neigh_event_send(neigh, NULL);2079 if (neigh)2080 neigh_release(neigh);2081exit:2082 dst_release(dst);2083 2084 return rc;2085}2086 2087/**2088 * irdma_find_node - find a cm node that matches the reference cm node2089 * @cm_core: cm's core2090 * @rem_port: remote tcp port num2091 * @rem_addr: remote ip addr2092 * @loc_port: local tcp port num2093 * @loc_addr: local ip addr2094 * @vlan_id: local VLAN ID2095 */2096struct irdma_cm_node *irdma_find_node(struct irdma_cm_core *cm_core,2097 u16 rem_port, u32 *rem_addr, u16 loc_port,2098 u32 *loc_addr, u16 vlan_id)2099{2100 struct irdma_cm_node *cm_node;2101 u32 key = (rem_port << 16) | loc_port;2102 2103 rcu_read_lock();2104 hash_for_each_possible_rcu(cm_core->cm_hash_tbl, cm_node, list, key) {2105 if (cm_node->vlan_id == vlan_id &&2106 cm_node->loc_port == loc_port && cm_node->rem_port == rem_port &&2107 !memcmp(cm_node->loc_addr, loc_addr, sizeof(cm_node->loc_addr)) &&2108 !memcmp(cm_node->rem_addr, rem_addr, sizeof(cm_node->rem_addr))) {2109 if (!refcount_inc_not_zero(&cm_node->refcnt))2110 goto exit;2111 rcu_read_unlock();2112 trace_irdma_find_node(cm_node, 0, NULL);2113 return cm_node;2114 }2115 }2116 2117exit:2118 rcu_read_unlock();2119 2120 /* no owner node */2121 return NULL;2122}2123 2124/**2125 * irdma_add_hte_node - add a cm node to the hash table2126 * @cm_core: cm's core2127 * @cm_node: connection's node2128 */2129static void irdma_add_hte_node(struct irdma_cm_core *cm_core,2130 struct irdma_cm_node *cm_node)2131{2132 unsigned long flags;2133 u32 key = (cm_node->rem_port << 16) | cm_node->loc_port;2134 2135 spin_lock_irqsave(&cm_core->ht_lock, flags);2136 hash_add_rcu(cm_core->cm_hash_tbl, &cm_node->list, key);2137 spin_unlock_irqrestore(&cm_core->ht_lock, flags);2138}2139 2140/**2141 * irdma_ipv4_is_lpb - check if loopback2142 * @loc_addr: local addr to compare2143 * @rem_addr: remote address2144 */2145bool irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr)2146{2147 return ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr);2148}2149 2150/**2151 * irdma_ipv6_is_lpb - check if loopback2152 * @loc_addr: local addr to compare2153 * @rem_addr: remote address2154 */2155bool irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr)2156{2157 struct in6_addr raddr6;2158 2159 irdma_copy_ip_htonl(raddr6.in6_u.u6_addr32, rem_addr);2160 2161 return !memcmp(loc_addr, rem_addr, 16) || ipv6_addr_loopback(&raddr6);2162}2163 2164/**2165 * irdma_cm_create_ah - create a cm address handle2166 * @cm_node: The connection manager node to create AH for2167 * @wait: Provides option to wait for ah creation or not2168 */2169static int irdma_cm_create_ah(struct irdma_cm_node *cm_node, bool wait)2170{2171 struct irdma_ah_info ah_info = {};2172 struct irdma_device *iwdev = cm_node->iwdev;2173 2174 ether_addr_copy(ah_info.mac_addr, iwdev->netdev->dev_addr);2175 2176 ah_info.hop_ttl = 0x40;2177 ah_info.tc_tos = cm_node->tos;2178 ah_info.vsi = &iwdev->vsi;2179 2180 if (cm_node->ipv4) {2181 ah_info.ipv4_valid = true;2182 ah_info.dest_ip_addr[0] = cm_node->rem_addr[0];2183 ah_info.src_ip_addr[0] = cm_node->loc_addr[0];2184 ah_info.do_lpbk = irdma_ipv4_is_lpb(ah_info.src_ip_addr[0],2185 ah_info.dest_ip_addr[0]);2186 } else {2187 memcpy(ah_info.dest_ip_addr, cm_node->rem_addr,2188 sizeof(ah_info.dest_ip_addr));2189 memcpy(ah_info.src_ip_addr, cm_node->loc_addr,2190 sizeof(ah_info.src_ip_addr));2191 ah_info.do_lpbk = irdma_ipv6_is_lpb(ah_info.src_ip_addr,2192 ah_info.dest_ip_addr);2193 }2194 2195 ah_info.vlan_tag = cm_node->vlan_id;2196 if (cm_node->vlan_id < VLAN_N_VID) {2197 ah_info.insert_vlan_tag = 1;2198 ah_info.vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT;2199 }2200 2201 ah_info.dst_arpindex =2202 irdma_arp_table(iwdev->rf, ah_info.dest_ip_addr,2203 ah_info.ipv4_valid, NULL, IRDMA_ARP_RESOLVE);2204 2205 if (irdma_puda_create_ah(&iwdev->rf->sc_dev, &ah_info, wait,2206 IRDMA_PUDA_RSRC_TYPE_ILQ, cm_node,2207 &cm_node->ah))2208 return -ENOMEM;2209 2210 trace_irdma_create_ah(cm_node);2211 return 0;2212}2213 2214/**2215 * irdma_cm_free_ah - free a cm address handle2216 * @cm_node: The connection manager node to create AH for2217 */2218static void irdma_cm_free_ah(struct irdma_cm_node *cm_node)2219{2220 struct irdma_device *iwdev = cm_node->iwdev;2221 2222 trace_irdma_cm_free_ah(cm_node);2223 irdma_puda_free_ah(&iwdev->rf->sc_dev, cm_node->ah);2224 cm_node->ah = NULL;2225}2226 2227/**2228 * irdma_make_cm_node - create a new instance of a cm node2229 * @cm_core: cm's core2230 * @iwdev: iwarp device structure2231 * @cm_info: quad info for connection2232 * @listener: passive connection's listener2233 */2234static struct irdma_cm_node *2235irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev,2236 struct irdma_cm_info *cm_info,2237 struct irdma_cm_listener *listener)2238{2239 struct irdma_cm_node *cm_node;2240 int oldarpindex;2241 int arpindex;2242 struct net_device *netdev = iwdev->netdev;2243 2244 /* create an hte and cm_node for this instance */2245 cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC);2246 if (!cm_node)2247 return NULL;2248 2249 /* set our node specific transport info */2250 cm_node->ipv4 = cm_info->ipv4;2251 cm_node->vlan_id = cm_info->vlan_id;2252 if (cm_node->vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)2253 cm_node->vlan_id = 0;2254 cm_node->tos = cm_info->tos;2255 cm_node->user_pri = cm_info->user_pri;2256 if (listener) {2257 if (listener->tos != cm_info->tos)2258 ibdev_warn(&iwdev->ibdev,2259 "application TOS[%d] and remote client TOS[%d] mismatch\n",2260 listener->tos, cm_info->tos);2261 if (iwdev->vsi.dscp_mode) {2262 cm_node->user_pri = listener->user_pri;2263 } else {2264 cm_node->tos = max(listener->tos, cm_info->tos);2265 cm_node->user_pri = rt_tos2priority(cm_node->tos);2266 cm_node->user_pri =2267 irdma_iw_get_vlan_prio(cm_info->loc_addr,2268 cm_node->user_pri,2269 cm_info->ipv4);2270 }2271 ibdev_dbg(&iwdev->ibdev,2272 "DCB: listener: TOS:[%d] UP:[%d]\n", cm_node->tos,2273 cm_node->user_pri);2274 trace_irdma_listener_tos(iwdev, cm_node->tos,2275 cm_node->user_pri);2276 }2277 memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr));2278 memcpy(cm_node->rem_addr, cm_info->rem_addr, sizeof(cm_node->rem_addr));2279 cm_node->loc_port = cm_info->loc_port;2280 cm_node->rem_port = cm_info->rem_port;2281 2282 cm_node->mpa_frame_rev = IRDMA_CM_DEFAULT_MPA_VER;2283 cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;2284 cm_node->iwdev = iwdev;2285 cm_node->dev = &iwdev->rf->sc_dev;2286 2287 cm_node->ird_size = cm_node->dev->hw_attrs.max_hw_ird;2288 cm_node->ord_size = cm_node->dev->hw_attrs.max_hw_ord;2289 2290 cm_node->listener = listener;2291 cm_node->cm_id = cm_info->cm_id;2292 ether_addr_copy(cm_node->loc_mac, netdev->dev_addr);2293 spin_lock_init(&cm_node->retrans_list_lock);2294 cm_node->ack_rcvd = false;2295 2296 init_completion(&cm_node->establish_comp);2297 refcount_set(&cm_node->refcnt, 1);2298 /* associate our parent CM core */2299 cm_node->cm_core = cm_core;2300 cm_node->tcp_cntxt.loc_id = IRDMA_CM_DEFAULT_LOCAL_ID;2301 cm_node->tcp_cntxt.rcv_wscale = iwdev->rcv_wscale;2302 cm_node->tcp_cntxt.rcv_wnd = iwdev->rcv_wnd >> cm_node->tcp_cntxt.rcv_wscale;2303 if (cm_node->ipv4) {2304 cm_node->tcp_cntxt.loc_seq_num = secure_tcp_seq(htonl(cm_node->loc_addr[0]),2305 htonl(cm_node->rem_addr[0]),2306 htons(cm_node->loc_port),2307 htons(cm_node->rem_port));2308 cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV4;2309 } else if (IS_ENABLED(CONFIG_IPV6)) {2310 __be32 loc[4] = {2311 htonl(cm_node->loc_addr[0]), htonl(cm_node->loc_addr[1]),2312 htonl(cm_node->loc_addr[2]), htonl(cm_node->loc_addr[3])2313 };2314 __be32 rem[4] = {2315 htonl(cm_node->rem_addr[0]), htonl(cm_node->rem_addr[1]),2316 htonl(cm_node->rem_addr[2]), htonl(cm_node->rem_addr[3])2317 };2318 cm_node->tcp_cntxt.loc_seq_num = secure_tcpv6_seq(loc, rem,2319 htons(cm_node->loc_port),2320 htons(cm_node->rem_port));2321 cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6;2322 }2323 2324 if ((cm_node->ipv4 &&2325 irdma_ipv4_is_lpb(cm_node->loc_addr[0], cm_node->rem_addr[0])) ||2326 (!cm_node->ipv4 &&2327 irdma_ipv6_is_lpb(cm_node->loc_addr, cm_node->rem_addr))) {2328 cm_node->do_lpb = true;2329 arpindex = irdma_arp_table(iwdev->rf, cm_node->rem_addr,2330 cm_node->ipv4, NULL,2331 IRDMA_ARP_RESOLVE);2332 } else {2333 oldarpindex = irdma_arp_table(iwdev->rf, cm_node->rem_addr,2334 cm_node->ipv4, NULL,2335 IRDMA_ARP_RESOLVE);2336 if (cm_node->ipv4)2337 arpindex = irdma_addr_resolve_neigh(iwdev,2338 cm_info->loc_addr[0],2339 cm_info->rem_addr[0],2340 oldarpindex);2341 else if (IS_ENABLED(CONFIG_IPV6))2342 arpindex = irdma_addr_resolve_neigh_ipv6(iwdev,2343 cm_info->loc_addr,2344 cm_info->rem_addr,2345 oldarpindex);2346 else2347 arpindex = -EINVAL;2348 }2349 2350 if (arpindex < 0)2351 goto err;2352 2353 ether_addr_copy(cm_node->rem_mac,2354 iwdev->rf->arp_table[arpindex].mac_addr);2355 irdma_add_hte_node(cm_core, cm_node);2356 cm_core->stats_nodes_created++;2357 return cm_node;2358 2359err:2360 kfree(cm_node);2361 2362 return NULL;2363}2364 2365static void irdma_destroy_connection(struct irdma_cm_node *cm_node)2366{2367 struct irdma_cm_core *cm_core = cm_node->cm_core;2368 struct irdma_qp *iwqp;2369 struct irdma_cm_info nfo;2370 2371 /* if the node is destroyed before connection was accelerated */2372 if (!cm_node->accelerated && cm_node->accept_pend) {2373 ibdev_dbg(&cm_node->iwdev->ibdev,2374 "CM: node destroyed before established\n");2375 atomic_dec(&cm_node->listener->pend_accepts_cnt);2376 }2377 if (cm_node->close_entry)2378 irdma_handle_close_entry(cm_node, 0);2379 if (cm_node->listener) {2380 irdma_dec_refcnt_listen(cm_core, cm_node->listener, 0, true);2381 } else {2382 if (cm_node->apbvt_set) {2383 irdma_del_apbvt(cm_node->iwdev, cm_node->apbvt_entry);2384 cm_node->apbvt_set = 0;2385 }2386 irdma_get_addr_info(cm_node, &nfo);2387 if (cm_node->qhash_set) {2388 nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;2389 irdma_manage_qhash(cm_node->iwdev, &nfo,2390 IRDMA_QHASH_TYPE_TCP_ESTABLISHED,2391 IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL,2392 false);2393 cm_node->qhash_set = 0;2394 }2395 }2396 2397 iwqp = cm_node->iwqp;2398 if (iwqp) {2399 cm_node->cm_id->rem_ref(cm_node->cm_id);2400 cm_node->cm_id = NULL;2401 iwqp->cm_id = NULL;2402 irdma_qp_rem_ref(&iwqp->ibqp);2403 cm_node->iwqp = NULL;2404 } else if (cm_node->qhash_set) {2405 irdma_get_addr_info(cm_node, &nfo);2406 nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;2407 irdma_manage_qhash(cm_node->iwdev, &nfo,2408 IRDMA_QHASH_TYPE_TCP_ESTABLISHED,2409 IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL, false);2410 cm_node->qhash_set = 0;2411 }2412 2413 cm_core->cm_free_ah(cm_node);2414}2415 2416/**2417 * irdma_rem_ref_cm_node - destroy an instance of a cm node2418 * @cm_node: connection's node2419 */2420void irdma_rem_ref_cm_node(struct irdma_cm_node *cm_node)2421{2422 struct irdma_cm_core *cm_core = cm_node->cm_core;2423 unsigned long flags;2424 2425 trace_irdma_rem_ref_cm_node(cm_node, 0, __builtin_return_address(0));2426 spin_lock_irqsave(&cm_core->ht_lock, flags);2427 2428 if (!refcount_dec_and_test(&cm_node->refcnt)) {2429 spin_unlock_irqrestore(&cm_core->ht_lock, flags);2430 return;2431 }2432 if (cm_node->iwqp) {2433 cm_node->iwqp->cm_node = NULL;2434 cm_node->iwqp->cm_id = NULL;2435 }2436 hash_del_rcu(&cm_node->list);2437 cm_node->cm_core->stats_nodes_destroyed++;2438 2439 spin_unlock_irqrestore(&cm_core->ht_lock, flags);2440 2441 irdma_destroy_connection(cm_node);2442 2443 kfree_rcu(cm_node, rcu_head);2444}2445 2446/**2447 * irdma_handle_fin_pkt - FIN packet received2448 * @cm_node: connection's node2449 */2450static void irdma_handle_fin_pkt(struct irdma_cm_node *cm_node)2451{2452 switch (cm_node->state) {2453 case IRDMA_CM_STATE_SYN_RCVD:2454 case IRDMA_CM_STATE_SYN_SENT:2455 case IRDMA_CM_STATE_ESTABLISHED:2456 case IRDMA_CM_STATE_MPAREJ_RCVD:2457 cm_node->tcp_cntxt.rcv_nxt++;2458 irdma_cleanup_retrans_entry(cm_node);2459 cm_node->state = IRDMA_CM_STATE_LAST_ACK;2460 irdma_send_fin(cm_node);2461 break;2462 case IRDMA_CM_STATE_MPAREQ_SENT:2463 irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);2464 cm_node->tcp_cntxt.rcv_nxt++;2465 irdma_cleanup_retrans_entry(cm_node);2466 cm_node->state = IRDMA_CM_STATE_CLOSED;2467 refcount_inc(&cm_node->refcnt);2468 irdma_send_reset(cm_node);2469 break;2470 case IRDMA_CM_STATE_FIN_WAIT1:2471 cm_node->tcp_cntxt.rcv_nxt++;2472 irdma_cleanup_retrans_entry(cm_node);2473 cm_node->state = IRDMA_CM_STATE_CLOSING;2474 irdma_send_ack(cm_node);2475 /*2476 * Wait for ACK as this is simultaneous close.2477 * After we receive ACK, do not send anything.2478 * Just rm the node.2479 */2480 break;2481 case IRDMA_CM_STATE_FIN_WAIT2:2482 cm_node->tcp_cntxt.rcv_nxt++;2483 irdma_cleanup_retrans_entry(cm_node);2484 cm_node->state = IRDMA_CM_STATE_TIME_WAIT;2485 irdma_send_ack(cm_node);2486 irdma_schedule_cm_timer(cm_node, NULL, IRDMA_TIMER_TYPE_CLOSE,2487 1, 0);2488 break;2489 case IRDMA_CM_STATE_TIME_WAIT:2490 cm_node->tcp_cntxt.rcv_nxt++;2491 irdma_cleanup_retrans_entry(cm_node);2492 cm_node->state = IRDMA_CM_STATE_CLOSED;2493 irdma_rem_ref_cm_node(cm_node);2494 break;2495 case IRDMA_CM_STATE_OFFLOADED:2496 default:2497 ibdev_dbg(&cm_node->iwdev->ibdev,2498 "CM: bad state node state = %d\n", cm_node->state);2499 break;2500 }2501}2502 2503/**2504 * irdma_handle_rst_pkt - process received RST packet2505 * @cm_node: connection's node2506 * @rbuf: receive buffer2507 */2508static void irdma_handle_rst_pkt(struct irdma_cm_node *cm_node,2509 struct irdma_puda_buf *rbuf)2510{2511 ibdev_dbg(&cm_node->iwdev->ibdev,2512 "CM: caller: %pS cm_node=%p state=%d rem_port=0x%04x loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n",2513 __builtin_return_address(0), cm_node, cm_node->state,2514 cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,2515 cm_node->loc_addr);2516 2517 irdma_cleanup_retrans_entry(cm_node);2518 switch (cm_node->state) {2519 case IRDMA_CM_STATE_SYN_SENT:2520 case IRDMA_CM_STATE_MPAREQ_SENT:2521 switch (cm_node->mpa_frame_rev) {2522 case IETF_MPA_V2:2523 /* Drop down to MPA_V1*/2524 cm_node->mpa_frame_rev = IETF_MPA_V1;2525 /* send a syn and goto syn sent state */2526 cm_node->state = IRDMA_CM_STATE_SYN_SENT;2527 if (irdma_send_syn(cm_node, 0))2528 irdma_active_open_err(cm_node, false);2529 break;2530 case IETF_MPA_V1:2531 default:2532 irdma_active_open_err(cm_node, false);2533 break;2534 }2535 break;2536 case IRDMA_CM_STATE_MPAREQ_RCVD:2537 atomic_inc(&cm_node->passive_state);2538 break;2539 case IRDMA_CM_STATE_ESTABLISHED:2540 case IRDMA_CM_STATE_SYN_RCVD:2541 case IRDMA_CM_STATE_LISTENING:2542 irdma_passive_open_err(cm_node, false);2543 break;2544 case IRDMA_CM_STATE_OFFLOADED:2545 irdma_active_open_err(cm_node, false);2546 break;2547 case IRDMA_CM_STATE_CLOSED:2548 break;2549 case IRDMA_CM_STATE_FIN_WAIT2:2550 case IRDMA_CM_STATE_FIN_WAIT1:2551 case IRDMA_CM_STATE_LAST_ACK:2552 case IRDMA_CM_STATE_TIME_WAIT:2553 cm_node->state = IRDMA_CM_STATE_CLOSED;2554 irdma_rem_ref_cm_node(cm_node);2555 break;2556 default:2557 break;2558 }2559}2560 2561/**2562 * irdma_handle_rcv_mpa - Process a recv'd mpa buffer2563 * @cm_node: connection's node2564 * @rbuf: receive buffer2565 */2566static void irdma_handle_rcv_mpa(struct irdma_cm_node *cm_node,2567 struct irdma_puda_buf *rbuf)2568{2569 int err;2570 int datasize = rbuf->datalen;2571 u8 *dataloc = rbuf->data;2572 2573 enum irdma_cm_event_type type = IRDMA_CM_EVENT_UNKNOWN;2574 u32 res_type;2575 2576 err = irdma_parse_mpa(cm_node, dataloc, &res_type, datasize);2577 if (err) {2578 if (cm_node->state == IRDMA_CM_STATE_MPAREQ_SENT)2579 irdma_active_open_err(cm_node, true);2580 else2581 irdma_passive_open_err(cm_node, true);2582 return;2583 }2584 2585 switch (cm_node->state) {2586 case IRDMA_CM_STATE_ESTABLISHED:2587 if (res_type == IRDMA_MPA_REQUEST_REJECT)2588 ibdev_dbg(&cm_node->iwdev->ibdev,2589 "CM: state for reject\n");2590 cm_node->state = IRDMA_CM_STATE_MPAREQ_RCVD;2591 type = IRDMA_CM_EVENT_MPA_REQ;2592 irdma_send_ack(cm_node); /* ACK received MPA request */2593 atomic_set(&cm_node->passive_state,2594 IRDMA_PASSIVE_STATE_INDICATED);2595 break;2596 case IRDMA_CM_STATE_MPAREQ_SENT:2597 irdma_cleanup_retrans_entry(cm_node);2598 if (res_type == IRDMA_MPA_REQUEST_REJECT) {2599 type = IRDMA_CM_EVENT_MPA_REJECT;2600 cm_node->state = IRDMA_CM_STATE_MPAREJ_RCVD;2601 } else {2602 type = IRDMA_CM_EVENT_CONNECTED;2603 cm_node->state = IRDMA_CM_STATE_OFFLOADED;2604 }2605 irdma_send_ack(cm_node);2606 break;2607 default:2608 ibdev_dbg(&cm_node->iwdev->ibdev,2609 "CM: wrong cm_node state =%d\n", cm_node->state);2610 break;2611 }2612 irdma_create_event(cm_node, type);2613}2614 2615/**2616 * irdma_check_syn - Check for error on received syn ack2617 * @cm_node: connection's node2618 * @tcph: pointer tcp header2619 */2620static int irdma_check_syn(struct irdma_cm_node *cm_node, struct tcphdr *tcph)2621{2622 if (ntohl(tcph->ack_seq) != cm_node->tcp_cntxt.loc_seq_num) {2623 irdma_active_open_err(cm_node, true);2624 return 1;2625 }2626 2627 return 0;2628}2629 2630/**2631 * irdma_check_seq - check seq numbers if OK2632 * @cm_node: connection's node2633 * @tcph: pointer tcp header2634 */2635static int irdma_check_seq(struct irdma_cm_node *cm_node, struct tcphdr *tcph)2636{2637 u32 seq;2638 u32 ack_seq;2639 u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num;2640 u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt;2641 u32 rcv_wnd;2642 int err = 0;2643 2644 seq = ntohl(tcph->seq);2645 ack_seq = ntohl(tcph->ack_seq);2646 rcv_wnd = cm_node->tcp_cntxt.rcv_wnd;2647 if (ack_seq != loc_seq_num ||2648 !between(seq, rcv_nxt, (rcv_nxt + rcv_wnd)))2649 err = -1;2650 if (err)2651 ibdev_dbg(&cm_node->iwdev->ibdev,2652 "CM: seq number err\n");2653 2654 return err;2655}2656 2657void irdma_add_conn_est_qh(struct irdma_cm_node *cm_node)2658{2659 struct irdma_cm_info nfo;2660 2661 irdma_get_addr_info(cm_node, &nfo);2662 nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;2663 irdma_manage_qhash(cm_node->iwdev, &nfo,2664 IRDMA_QHASH_TYPE_TCP_ESTABLISHED,2665 IRDMA_QHASH_MANAGE_TYPE_ADD,2666 cm_node, false);2667 cm_node->qhash_set = true;2668}2669 2670/**2671 * irdma_handle_syn_pkt - is for Passive node2672 * @cm_node: connection's node2673 * @rbuf: receive buffer2674 */2675static void irdma_handle_syn_pkt(struct irdma_cm_node *cm_node,2676 struct irdma_puda_buf *rbuf)2677{2678 struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;2679 int err;2680 u32 inc_sequence;2681 int optionsize;2682 2683 optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);2684 inc_sequence = ntohl(tcph->seq);2685 2686 switch (cm_node->state) {2687 case IRDMA_CM_STATE_SYN_SENT:2688 case IRDMA_CM_STATE_MPAREQ_SENT:2689 /* Rcvd syn on active open connection */2690 irdma_active_open_err(cm_node, 1);2691 break;2692 case IRDMA_CM_STATE_LISTENING:2693 /* Passive OPEN */2694 if (atomic_read(&cm_node->listener->pend_accepts_cnt) >2695 cm_node->listener->backlog) {2696 cm_node->cm_core->stats_backlog_drops++;2697 irdma_passive_open_err(cm_node, false);2698 break;2699 }2700 err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1);2701 if (err) {2702 irdma_passive_open_err(cm_node, false);2703 /* drop pkt */2704 break;2705 }2706 err = cm_node->cm_core->cm_create_ah(cm_node, false);2707 if (err) {2708 irdma_passive_open_err(cm_node, false);2709 /* drop pkt */2710 break;2711 }2712 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;2713 cm_node->accept_pend = 1;2714 atomic_inc(&cm_node->listener->pend_accepts_cnt);2715 2716 cm_node->state = IRDMA_CM_STATE_SYN_RCVD;2717 break;2718 case IRDMA_CM_STATE_CLOSED:2719 irdma_cleanup_retrans_entry(cm_node);2720 refcount_inc(&cm_node->refcnt);2721 irdma_send_reset(cm_node);2722 break;2723 case IRDMA_CM_STATE_OFFLOADED:2724 case IRDMA_CM_STATE_ESTABLISHED:2725 case IRDMA_CM_STATE_FIN_WAIT1:2726 case IRDMA_CM_STATE_FIN_WAIT2:2727 case IRDMA_CM_STATE_MPAREQ_RCVD:2728 case IRDMA_CM_STATE_LAST_ACK:2729 case IRDMA_CM_STATE_CLOSING:2730 case IRDMA_CM_STATE_UNKNOWN:2731 default:2732 break;2733 }2734}2735 2736/**2737 * irdma_handle_synack_pkt - Process SYN+ACK packet (active side)2738 * @cm_node: connection's node2739 * @rbuf: receive buffer2740 */2741static void irdma_handle_synack_pkt(struct irdma_cm_node *cm_node,2742 struct irdma_puda_buf *rbuf)2743{2744 struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;2745 int err;2746 u32 inc_sequence;2747 int optionsize;2748 2749 optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);2750 inc_sequence = ntohl(tcph->seq);2751 switch (cm_node->state) {2752 case IRDMA_CM_STATE_SYN_SENT:2753 irdma_cleanup_retrans_entry(cm_node);2754 /* active open */2755 if (irdma_check_syn(cm_node, tcph)) {2756 ibdev_dbg(&cm_node->iwdev->ibdev,2757 "CM: check syn fail\n");2758 return;2759 }2760 cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);2761 /* setup options */2762 err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 0);2763 if (err) {2764 ibdev_dbg(&cm_node->iwdev->ibdev,2765 "CM: cm_node=%p tcp_options failed\n",2766 cm_node);2767 break;2768 }2769 irdma_cleanup_retrans_entry(cm_node);2770 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;2771 irdma_send_ack(cm_node); /* ACK for the syn_ack */2772 err = irdma_send_mpa_request(cm_node);2773 if (err) {2774 ibdev_dbg(&cm_node->iwdev->ibdev,2775 "CM: cm_node=%p irdma_send_mpa_request failed\n",2776 cm_node);2777 break;2778 }2779 cm_node->state = IRDMA_CM_STATE_MPAREQ_SENT;2780 break;2781 case IRDMA_CM_STATE_MPAREQ_RCVD:2782 irdma_passive_open_err(cm_node, true);2783 break;2784 case IRDMA_CM_STATE_LISTENING:2785 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);2786 irdma_cleanup_retrans_entry(cm_node);2787 cm_node->state = IRDMA_CM_STATE_CLOSED;2788 irdma_send_reset(cm_node);2789 break;2790 case IRDMA_CM_STATE_CLOSED:2791 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);2792 irdma_cleanup_retrans_entry(cm_node);2793 refcount_inc(&cm_node->refcnt);2794 irdma_send_reset(cm_node);2795 break;2796 case IRDMA_CM_STATE_ESTABLISHED:2797 case IRDMA_CM_STATE_FIN_WAIT1:2798 case IRDMA_CM_STATE_FIN_WAIT2:2799 case IRDMA_CM_STATE_LAST_ACK:2800 case IRDMA_CM_STATE_OFFLOADED:2801 case IRDMA_CM_STATE_CLOSING:2802 case IRDMA_CM_STATE_UNKNOWN:2803 case IRDMA_CM_STATE_MPAREQ_SENT:2804 default:2805 break;2806 }2807}2808 2809/**2810 * irdma_handle_ack_pkt - process packet with ACK2811 * @cm_node: connection's node2812 * @rbuf: receive buffer2813 */2814static int irdma_handle_ack_pkt(struct irdma_cm_node *cm_node,2815 struct irdma_puda_buf *rbuf)2816{2817 struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;2818 u32 inc_sequence;2819 int ret;2820 int optionsize;2821 u32 datasize = rbuf->datalen;2822 2823 optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);2824 2825 if (irdma_check_seq(cm_node, tcph))2826 return -EINVAL;2827 2828 inc_sequence = ntohl(tcph->seq);2829 switch (cm_node->state) {2830 case IRDMA_CM_STATE_SYN_RCVD:2831 irdma_cleanup_retrans_entry(cm_node);2832 ret = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1);2833 if (ret)2834 return ret;2835 cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);2836 cm_node->state = IRDMA_CM_STATE_ESTABLISHED;2837 if (datasize) {2838 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;2839 irdma_handle_rcv_mpa(cm_node, rbuf);2840 }2841 break;2842 case IRDMA_CM_STATE_ESTABLISHED:2843 irdma_cleanup_retrans_entry(cm_node);2844 if (datasize) {2845 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;2846 irdma_handle_rcv_mpa(cm_node, rbuf);2847 }2848 break;2849 case IRDMA_CM_STATE_MPAREQ_SENT:2850 cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);2851 if (datasize) {2852 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;2853 cm_node->ack_rcvd = false;2854 irdma_handle_rcv_mpa(cm_node, rbuf);2855 } else {2856 cm_node->ack_rcvd = true;2857 }2858 break;2859 case IRDMA_CM_STATE_LISTENING:2860 irdma_cleanup_retrans_entry(cm_node);2861 cm_node->state = IRDMA_CM_STATE_CLOSED;2862 irdma_send_reset(cm_node);2863 break;2864 case IRDMA_CM_STATE_CLOSED:2865 irdma_cleanup_retrans_entry(cm_node);2866 refcount_inc(&cm_node->refcnt);2867 irdma_send_reset(cm_node);2868 break;2869 case IRDMA_CM_STATE_LAST_ACK:2870 case IRDMA_CM_STATE_CLOSING:2871 irdma_cleanup_retrans_entry(cm_node);2872 cm_node->state = IRDMA_CM_STATE_CLOSED;2873 irdma_rem_ref_cm_node(cm_node);2874 break;2875 case IRDMA_CM_STATE_FIN_WAIT1:2876 irdma_cleanup_retrans_entry(cm_node);2877 cm_node->state = IRDMA_CM_STATE_FIN_WAIT2;2878 break;2879 case IRDMA_CM_STATE_SYN_SENT:2880 case IRDMA_CM_STATE_FIN_WAIT2:2881 case IRDMA_CM_STATE_OFFLOADED:2882 case IRDMA_CM_STATE_MPAREQ_RCVD:2883 case IRDMA_CM_STATE_UNKNOWN:2884 default:2885 irdma_cleanup_retrans_entry(cm_node);2886 break;2887 }2888 2889 return 0;2890}2891 2892/**2893 * irdma_process_pkt - process cm packet2894 * @cm_node: connection's node2895 * @rbuf: receive buffer2896 */2897static void irdma_process_pkt(struct irdma_cm_node *cm_node,2898 struct irdma_puda_buf *rbuf)2899{2900 enum irdma_tcpip_pkt_type pkt_type = IRDMA_PKT_TYPE_UNKNOWN;2901 struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;2902 u32 fin_set = 0;2903 int err;2904 2905 if (tcph->rst) {2906 pkt_type = IRDMA_PKT_TYPE_RST;2907 } else if (tcph->syn) {2908 pkt_type = IRDMA_PKT_TYPE_SYN;2909 if (tcph->ack)2910 pkt_type = IRDMA_PKT_TYPE_SYNACK;2911 } else if (tcph->ack) {2912 pkt_type = IRDMA_PKT_TYPE_ACK;2913 }2914 if (tcph->fin)2915 fin_set = 1;2916 2917 switch (pkt_type) {2918 case IRDMA_PKT_TYPE_SYN:2919 irdma_handle_syn_pkt(cm_node, rbuf);2920 break;2921 case IRDMA_PKT_TYPE_SYNACK:2922 irdma_handle_synack_pkt(cm_node, rbuf);2923 break;2924 case IRDMA_PKT_TYPE_ACK:2925 err = irdma_handle_ack_pkt(cm_node, rbuf);2926 if (fin_set && !err)2927 irdma_handle_fin_pkt(cm_node);2928 break;2929 case IRDMA_PKT_TYPE_RST:2930 irdma_handle_rst_pkt(cm_node, rbuf);2931 break;2932 default:2933 if (fin_set &&2934 (!irdma_check_seq(cm_node, (struct tcphdr *)rbuf->tcph)))2935 irdma_handle_fin_pkt(cm_node);2936 break;2937 }2938}2939 2940/**2941 * irdma_make_listen_node - create a listen node with params2942 * @cm_core: cm's core2943 * @iwdev: iwarp device structure2944 * @cm_info: quad info for connection2945 */2946static struct irdma_cm_listener *2947irdma_make_listen_node(struct irdma_cm_core *cm_core,2948 struct irdma_device *iwdev,2949 struct irdma_cm_info *cm_info)2950{2951 struct irdma_cm_listener *listener;2952 unsigned long flags;2953 2954 /* cannot have multiple matching listeners */2955 listener =2956 irdma_find_listener(cm_core, cm_info->loc_addr, cm_info->ipv4,2957 cm_info->loc_port, cm_info->vlan_id,2958 IRDMA_CM_LISTENER_EITHER_STATE);2959 if (listener &&2960 listener->listener_state == IRDMA_CM_LISTENER_ACTIVE_STATE) {2961 refcount_dec(&listener->refcnt);2962 return NULL;2963 }2964 2965 if (!listener) {2966 /* create a CM listen node2967 * 1/2 node to compare incoming traffic to2968 */2969 listener = kzalloc(sizeof(*listener), GFP_KERNEL);2970 if (!listener)2971 return NULL;2972 cm_core->stats_listen_nodes_created++;2973 memcpy(listener->loc_addr, cm_info->loc_addr,2974 sizeof(listener->loc_addr));2975 listener->loc_port = cm_info->loc_port;2976 2977 INIT_LIST_HEAD(&listener->child_listen_list);2978 2979 refcount_set(&listener->refcnt, 1);2980 } else {2981 listener->reused_node = 1;2982 }2983 2984 listener->cm_id = cm_info->cm_id;2985 listener->ipv4 = cm_info->ipv4;2986 listener->vlan_id = cm_info->vlan_id;2987 atomic_set(&listener->pend_accepts_cnt, 0);2988 listener->cm_core = cm_core;2989 listener->iwdev = iwdev;2990 2991 listener->backlog = cm_info->backlog;2992 listener->listener_state = IRDMA_CM_LISTENER_ACTIVE_STATE;2993 2994 if (!listener->reused_node) {2995 spin_lock_irqsave(&cm_core->listen_list_lock, flags);2996 list_add(&listener->list, &cm_core->listen_list);2997 spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);2998 }2999 3000 return listener;3001}3002 3003/**3004 * irdma_create_cm_node - make a connection node with params3005 * @cm_core: cm's core3006 * @iwdev: iwarp device structure3007 * @conn_param: connection parameters3008 * @cm_info: quad info for connection3009 * @caller_cm_node: pointer to cm_node structure to return3010 */3011static int irdma_create_cm_node(struct irdma_cm_core *cm_core,3012 struct irdma_device *iwdev,3013 struct iw_cm_conn_param *conn_param,3014 struct irdma_cm_info *cm_info,3015 struct irdma_cm_node **caller_cm_node)3016{3017 struct irdma_cm_node *cm_node;3018 u16 private_data_len = conn_param->private_data_len;3019 const void *private_data = conn_param->private_data;3020 3021 /* create a CM connection node */3022 cm_node = irdma_make_cm_node(cm_core, iwdev, cm_info, NULL);3023 if (!cm_node)3024 return -ENOMEM;3025 3026 /* set our node side to client (active) side */3027 cm_node->tcp_cntxt.client = 1;3028 cm_node->tcp_cntxt.rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;3029 3030 irdma_record_ird_ord(cm_node, conn_param->ird, conn_param->ord);3031 3032 cm_node->pdata.size = private_data_len;3033 cm_node->pdata.addr = cm_node->pdata_buf;3034 3035 memcpy(cm_node->pdata_buf, private_data, private_data_len);3036 *caller_cm_node = cm_node;3037 3038 return 0;3039}3040 3041/**3042 * irdma_cm_reject - reject and teardown a connection3043 * @cm_node: connection's node3044 * @pdata: ptr to private data for reject3045 * @plen: size of private data3046 */3047static int irdma_cm_reject(struct irdma_cm_node *cm_node, const void *pdata,3048 u8 plen)3049{3050 int ret;3051 int passive_state;3052 3053 if (cm_node->tcp_cntxt.client)3054 return 0;3055 3056 irdma_cleanup_retrans_entry(cm_node);3057 3058 passive_state = atomic_add_return(1, &cm_node->passive_state);3059 if (passive_state == IRDMA_SEND_RESET_EVENT) {3060 cm_node->state = IRDMA_CM_STATE_CLOSED;3061 irdma_rem_ref_cm_node(cm_node);3062 return 0;3063 }3064 3065 if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) {3066 irdma_rem_ref_cm_node(cm_node);3067 return 0;3068 }3069 3070 ret = irdma_send_mpa_reject(cm_node, pdata, plen);3071 if (!ret)3072 return 0;3073 3074 cm_node->state = IRDMA_CM_STATE_CLOSED;3075 if (irdma_send_reset(cm_node))3076 ibdev_dbg(&cm_node->iwdev->ibdev,3077 "CM: send reset failed\n");3078 3079 return ret;3080}3081 3082/**3083 * irdma_cm_close - close of cm connection3084 * @cm_node: connection's node3085 */3086static int irdma_cm_close(struct irdma_cm_node *cm_node)3087{3088 switch (cm_node->state) {3089 case IRDMA_CM_STATE_SYN_RCVD:3090 case IRDMA_CM_STATE_SYN_SENT:3091 case IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED:3092 case IRDMA_CM_STATE_ESTABLISHED:3093 case IRDMA_CM_STATE_ACCEPTING:3094 case IRDMA_CM_STATE_MPAREQ_SENT:3095 case IRDMA_CM_STATE_MPAREQ_RCVD:3096 irdma_cleanup_retrans_entry(cm_node);3097 irdma_send_reset(cm_node);3098 break;3099 case IRDMA_CM_STATE_CLOSE_WAIT:3100 cm_node->state = IRDMA_CM_STATE_LAST_ACK;3101 irdma_send_fin(cm_node);3102 break;3103 case IRDMA_CM_STATE_FIN_WAIT1:3104 case IRDMA_CM_STATE_FIN_WAIT2:3105 case IRDMA_CM_STATE_LAST_ACK:3106 case IRDMA_CM_STATE_TIME_WAIT:3107 case IRDMA_CM_STATE_CLOSING:3108 return -EINVAL;3109 case IRDMA_CM_STATE_LISTENING:3110 irdma_cleanup_retrans_entry(cm_node);3111 irdma_send_reset(cm_node);3112 break;3113 case IRDMA_CM_STATE_MPAREJ_RCVD:3114 case IRDMA_CM_STATE_UNKNOWN:3115 case IRDMA_CM_STATE_INITED:3116 case IRDMA_CM_STATE_CLOSED:3117 case IRDMA_CM_STATE_LISTENER_DESTROYED:3118 irdma_rem_ref_cm_node(cm_node);3119 break;3120 case IRDMA_CM_STATE_OFFLOADED:3121 if (cm_node->send_entry)3122 ibdev_dbg(&cm_node->iwdev->ibdev,3123 "CM: CM send_entry in OFFLOADED state\n");3124 irdma_rem_ref_cm_node(cm_node);3125 break;3126 }3127 3128 return 0;3129}3130 3131/**3132 * irdma_receive_ilq - recv an ETHERNET packet, and process it3133 * through CM3134 * @vsi: VSI structure of dev3135 * @rbuf: receive buffer3136 */3137void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf)3138{3139 struct irdma_cm_node *cm_node;3140 struct irdma_cm_listener *listener;3141 struct iphdr *iph;3142 struct ipv6hdr *ip6h;3143 struct tcphdr *tcph;3144 struct irdma_cm_info cm_info = {};3145 struct irdma_device *iwdev = vsi->back_vsi;3146 struct irdma_cm_core *cm_core = &iwdev->cm_core;3147 struct vlan_ethhdr *ethh;3148 u16 vtag;3149 3150 /* if vlan, then maclen = 18 else 14 */3151 iph = (struct iphdr *)rbuf->iph;3152 print_hex_dump_debug("ILQ: RECEIVE ILQ BUFFER", DUMP_PREFIX_OFFSET,3153 16, 8, rbuf->mem.va, rbuf->totallen, false);3154 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {3155 if (rbuf->vlan_valid) {3156 vtag = rbuf->vlan_id;3157 cm_info.user_pri = (vtag & VLAN_PRIO_MASK) >>3158 VLAN_PRIO_SHIFT;3159 cm_info.vlan_id = vtag & VLAN_VID_MASK;3160 } else {3161 cm_info.vlan_id = 0xFFFF;3162 }3163 } else {3164 ethh = rbuf->mem.va;3165 3166 if (ethh->h_vlan_proto == htons(ETH_P_8021Q)) {3167 vtag = ntohs(ethh->h_vlan_TCI);3168 cm_info.user_pri = (vtag & VLAN_PRIO_MASK) >>3169 VLAN_PRIO_SHIFT;3170 cm_info.vlan_id = vtag & VLAN_VID_MASK;3171 ibdev_dbg(&cm_core->iwdev->ibdev,3172 "CM: vlan_id=%d\n", cm_info.vlan_id);3173 } else {3174 cm_info.vlan_id = 0xFFFF;3175 }3176 }3177 tcph = (struct tcphdr *)rbuf->tcph;3178 3179 if (rbuf->ipv4) {3180 cm_info.loc_addr[0] = ntohl(iph->daddr);3181 cm_info.rem_addr[0] = ntohl(iph->saddr);3182 cm_info.ipv4 = true;3183 cm_info.tos = iph->tos;3184 } else {3185 ip6h = (struct ipv6hdr *)rbuf->iph;3186 irdma_copy_ip_ntohl(cm_info.loc_addr,3187 ip6h->daddr.in6_u.u6_addr32);3188 irdma_copy_ip_ntohl(cm_info.rem_addr,3189 ip6h->saddr.in6_u.u6_addr32);3190 cm_info.ipv4 = false;3191 cm_info.tos = (ip6h->priority << 4) | (ip6h->flow_lbl[0] >> 4);3192 }3193 cm_info.loc_port = ntohs(tcph->dest);3194 cm_info.rem_port = ntohs(tcph->source);3195 cm_node = irdma_find_node(cm_core, cm_info.rem_port, cm_info.rem_addr,3196 cm_info.loc_port, cm_info.loc_addr, cm_info.vlan_id);3197 3198 if (!cm_node) {3199 /* Only type of packet accepted are for the3200 * PASSIVE open (syn only)3201 */3202 if (!tcph->syn || tcph->ack)3203 return;3204 3205 listener = irdma_find_listener(cm_core,3206 cm_info.loc_addr,3207 cm_info.ipv4,3208 cm_info.loc_port,3209 cm_info.vlan_id,3210 IRDMA_CM_LISTENER_ACTIVE_STATE);3211 if (!listener) {3212 cm_info.cm_id = NULL;3213 ibdev_dbg(&cm_core->iwdev->ibdev,3214 "CM: no listener found\n");3215 return;3216 }3217 3218 cm_info.cm_id = listener->cm_id;3219 cm_node = irdma_make_cm_node(cm_core, iwdev, &cm_info,3220 listener);3221 if (!cm_node) {3222 ibdev_dbg(&cm_core->iwdev->ibdev,3223 "CM: allocate node failed\n");3224 refcount_dec(&listener->refcnt);3225 return;3226 }3227 3228 if (!tcph->rst && !tcph->fin) {3229 cm_node->state = IRDMA_CM_STATE_LISTENING;3230 } else {3231 irdma_rem_ref_cm_node(cm_node);3232 return;3233 }3234 3235 refcount_inc(&cm_node->refcnt);3236 } else if (cm_node->state == IRDMA_CM_STATE_OFFLOADED) {3237 irdma_rem_ref_cm_node(cm_node);3238 return;3239 }3240 3241 irdma_process_pkt(cm_node, rbuf);3242 irdma_rem_ref_cm_node(cm_node);3243}3244 3245static int irdma_add_qh(struct irdma_cm_node *cm_node, bool active)3246{3247 if (!active)3248 irdma_add_conn_est_qh(cm_node);3249 return 0;3250}3251 3252static void irdma_cm_free_ah_nop(struct irdma_cm_node *cm_node)3253{3254}3255 3256/**3257 * irdma_setup_cm_core - setup top level instance of a cm core3258 * @iwdev: iwarp device structure3259 * @rdma_ver: HW version3260 */3261int irdma_setup_cm_core(struct irdma_device *iwdev, u8 rdma_ver)3262{3263 struct irdma_cm_core *cm_core = &iwdev->cm_core;3264 3265 cm_core->iwdev = iwdev;3266 cm_core->dev = &iwdev->rf->sc_dev;3267 3268 /* Handles CM event work items send to Iwarp core */3269 cm_core->event_wq = alloc_ordered_workqueue("iwarp-event-wq", 0);3270 if (!cm_core->event_wq)3271 return -ENOMEM;3272 3273 INIT_LIST_HEAD(&cm_core->listen_list);3274 3275 timer_setup(&cm_core->tcp_timer, irdma_cm_timer_tick, 0);3276 3277 spin_lock_init(&cm_core->ht_lock);3278 spin_lock_init(&cm_core->listen_list_lock);3279 spin_lock_init(&cm_core->apbvt_lock);3280 switch (rdma_ver) {3281 case IRDMA_GEN_1:3282 cm_core->form_cm_frame = irdma_form_uda_cm_frame;3283 cm_core->cm_create_ah = irdma_add_qh;3284 cm_core->cm_free_ah = irdma_cm_free_ah_nop;3285 break;3286 case IRDMA_GEN_2:3287 default:3288 cm_core->form_cm_frame = irdma_form_ah_cm_frame;3289 cm_core->cm_create_ah = irdma_cm_create_ah;3290 cm_core->cm_free_ah = irdma_cm_free_ah;3291 }3292 3293 return 0;3294}3295 3296/**3297 * irdma_cleanup_cm_core - deallocate a top level instance of a3298 * cm core3299 * @cm_core: cm's core3300 */3301void irdma_cleanup_cm_core(struct irdma_cm_core *cm_core)3302{3303 if (!cm_core)3304 return;3305 3306 del_timer_sync(&cm_core->tcp_timer);3307 3308 destroy_workqueue(cm_core->event_wq);3309 cm_core->dev->ws_reset(&cm_core->iwdev->vsi);3310}3311 3312/**3313 * irdma_init_tcp_ctx - setup qp context3314 * @cm_node: connection's node3315 * @tcp_info: offload info for tcp3316 * @iwqp: associate qp for the connection3317 */3318static void irdma_init_tcp_ctx(struct irdma_cm_node *cm_node,3319 struct irdma_tcp_offload_info *tcp_info,3320 struct irdma_qp *iwqp)3321{3322 tcp_info->ipv4 = cm_node->ipv4;3323 tcp_info->drop_ooo_seg = !iwqp->iwdev->iw_ooo;3324 tcp_info->wscale = true;3325 tcp_info->ignore_tcp_opt = true;3326 tcp_info->ignore_tcp_uns_opt = true;3327 tcp_info->no_nagle = false;3328 3329 tcp_info->ttl = IRDMA_DEFAULT_TTL;3330 tcp_info->rtt_var = IRDMA_DEFAULT_RTT_VAR;3331 tcp_info->ss_thresh = IRDMA_DEFAULT_SS_THRESH;3332 tcp_info->rexmit_thresh = IRDMA_DEFAULT_REXMIT_THRESH;3333 3334 tcp_info->tcp_state = IRDMA_TCP_STATE_ESTABLISHED;3335 tcp_info->snd_wscale = cm_node->tcp_cntxt.snd_wscale;3336 tcp_info->rcv_wscale = cm_node->tcp_cntxt.rcv_wscale;3337 3338 tcp_info->snd_nxt = cm_node->tcp_cntxt.loc_seq_num;3339 tcp_info->snd_wnd = cm_node->tcp_cntxt.snd_wnd;3340 tcp_info->rcv_nxt = cm_node->tcp_cntxt.rcv_nxt;3341 tcp_info->snd_max = cm_node->tcp_cntxt.loc_seq_num;3342 3343 tcp_info->snd_una = cm_node->tcp_cntxt.loc_seq_num;3344 tcp_info->cwnd = 2 * cm_node->tcp_cntxt.mss;3345 tcp_info->snd_wl1 = cm_node->tcp_cntxt.rcv_nxt;3346 tcp_info->snd_wl2 = cm_node->tcp_cntxt.loc_seq_num;3347 tcp_info->max_snd_window = cm_node->tcp_cntxt.max_snd_wnd;3348 tcp_info->rcv_wnd = cm_node->tcp_cntxt.rcv_wnd3349 << cm_node->tcp_cntxt.rcv_wscale;3350 3351 tcp_info->flow_label = 0;3352 tcp_info->snd_mss = (u32)cm_node->tcp_cntxt.mss;3353 tcp_info->tos = cm_node->tos;3354 if (cm_node->vlan_id < VLAN_N_VID) {3355 tcp_info->insert_vlan_tag = true;3356 tcp_info->vlan_tag = cm_node->vlan_id;3357 tcp_info->vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT;3358 }3359 if (cm_node->ipv4) {3360 tcp_info->src_port = cm_node->loc_port;3361 tcp_info->dst_port = cm_node->rem_port;3362 3363 tcp_info->dest_ip_addr[3] = cm_node->rem_addr[0];3364 tcp_info->local_ipaddr[3] = cm_node->loc_addr[0];3365 tcp_info->arp_idx = (u16)irdma_arp_table(iwqp->iwdev->rf,3366 &tcp_info->dest_ip_addr[3],3367 true, NULL,3368 IRDMA_ARP_RESOLVE);3369 } else {3370 tcp_info->src_port = cm_node->loc_port;3371 tcp_info->dst_port = cm_node->rem_port;3372 memcpy(tcp_info->dest_ip_addr, cm_node->rem_addr,3373 sizeof(tcp_info->dest_ip_addr));3374 memcpy(tcp_info->local_ipaddr, cm_node->loc_addr,3375 sizeof(tcp_info->local_ipaddr));3376 3377 tcp_info->arp_idx = (u16)irdma_arp_table(iwqp->iwdev->rf,3378 &tcp_info->dest_ip_addr[0],3379 false, NULL,3380 IRDMA_ARP_RESOLVE);3381 }3382}3383 3384/**3385 * irdma_cm_init_tsa_conn - setup qp for RTS3386 * @iwqp: associate qp for the connection3387 * @cm_node: connection's node3388 */3389static void irdma_cm_init_tsa_conn(struct irdma_qp *iwqp,3390 struct irdma_cm_node *cm_node)3391{3392 struct irdma_iwarp_offload_info *iwarp_info;3393 struct irdma_qp_host_ctx_info *ctx_info;3394 3395 iwarp_info = &iwqp->iwarp_info;3396 ctx_info = &iwqp->ctx_info;3397 3398 ctx_info->tcp_info = &iwqp->tcp_info;3399 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;3400 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;3401 3402 iwarp_info->ord_size = cm_node->ord_size;3403 iwarp_info->ird_size = cm_node->ird_size;3404 iwarp_info->rd_en = true;3405 iwarp_info->rdmap_ver = 1;3406 iwarp_info->ddp_ver = 1;3407 iwarp_info->pd_id = iwqp->iwpd->sc_pd.pd_id;3408 3409 ctx_info->tcp_info_valid = true;3410 ctx_info->iwarp_info_valid = true;3411 ctx_info->user_pri = cm_node->user_pri;3412 3413 irdma_init_tcp_ctx(cm_node, &iwqp->tcp_info, iwqp);3414 if (cm_node->snd_mark_en) {3415 iwarp_info->snd_mark_en = true;3416 iwarp_info->snd_mark_offset = (iwqp->tcp_info.snd_nxt & SNDMARKER_SEQNMASK) +3417 cm_node->lsmm_size;3418 }3419 3420 cm_node->state = IRDMA_CM_STATE_OFFLOADED;3421 iwqp->tcp_info.tcp_state = IRDMA_TCP_STATE_ESTABLISHED;3422 iwqp->tcp_info.src_mac_addr_idx = iwqp->iwdev->mac_ip_table_idx;3423 3424 if (cm_node->rcv_mark_en) {3425 iwarp_info->rcv_mark_en = true;3426 iwarp_info->align_hdrs = true;3427 }3428 3429 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info);3430 3431 /* once tcp_info is set, no need to do it again */3432 ctx_info->tcp_info_valid = false;3433 ctx_info->iwarp_info_valid = false;3434}3435 3436/**3437 * irdma_cm_disconn - when a connection is being closed3438 * @iwqp: associated qp for the connection3439 */3440void irdma_cm_disconn(struct irdma_qp *iwqp)3441{3442 struct irdma_device *iwdev = iwqp->iwdev;3443 struct disconn_work *work;3444 unsigned long flags;3445 3446 work = kzalloc(sizeof(*work), GFP_ATOMIC);3447 if (!work)3448 return;3449 3450 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);3451 if (!iwdev->rf->qp_table[iwqp->ibqp.qp_num]) {3452 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);3453 ibdev_dbg(&iwdev->ibdev,3454 "CM: qp_id %d is already freed\n",3455 iwqp->ibqp.qp_num);3456 kfree(work);3457 return;3458 }3459 irdma_qp_add_ref(&iwqp->ibqp);3460 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);3461 3462 work->iwqp = iwqp;3463 INIT_WORK(&work->work, irdma_disconnect_worker);3464 queue_work(iwdev->cleanup_wq, &work->work);3465}3466 3467/**3468 * irdma_qp_disconnect - free qp and close cm3469 * @iwqp: associate qp for the connection3470 */3471static void irdma_qp_disconnect(struct irdma_qp *iwqp)3472{3473 struct irdma_device *iwdev = iwqp->iwdev;3474 3475 iwqp->active_conn = 0;3476 /* close the CM node down if it is still active */3477 ibdev_dbg(&iwdev->ibdev, "CM: Call close API\n");3478 irdma_cm_close(iwqp->cm_node);3479}3480 3481/**3482 * irdma_cm_disconn_true - called by worker thread to disconnect qp3483 * @iwqp: associate qp for the connection3484 */3485static void irdma_cm_disconn_true(struct irdma_qp *iwqp)3486{3487 struct iw_cm_id *cm_id;3488 struct irdma_device *iwdev;3489 struct irdma_sc_qp *qp = &iwqp->sc_qp;3490 u16 last_ae;3491 u8 original_hw_tcp_state;3492 u8 original_ibqp_state;3493 int disconn_status = 0;3494 int issue_disconn = 0;3495 int issue_close = 0;3496 int issue_flush = 0;3497 unsigned long flags;3498 int err;3499 3500 iwdev = iwqp->iwdev;3501 spin_lock_irqsave(&iwqp->lock, flags);3502 if (rdma_protocol_roce(&iwdev->ibdev, 1)) {3503 struct ib_qp_attr attr;3504 3505 if (iwqp->flush_issued || iwqp->sc_qp.qp_uk.destroy_pending) {3506 spin_unlock_irqrestore(&iwqp->lock, flags);3507 return;3508 }3509 3510 spin_unlock_irqrestore(&iwqp->lock, flags);3511 3512 attr.qp_state = IB_QPS_ERR;3513 irdma_modify_qp_roce(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);3514 irdma_ib_qp_event(iwqp, qp->event_type);3515 return;3516 }3517 3518 cm_id = iwqp->cm_id;3519 original_hw_tcp_state = iwqp->hw_tcp_state;3520 original_ibqp_state = iwqp->ibqp_state;3521 last_ae = iwqp->last_aeq;3522 3523 if (qp->term_flags) {3524 issue_disconn = 1;3525 issue_close = 1;3526 iwqp->cm_id = NULL;3527 irdma_terminate_del_timer(qp);3528 if (!iwqp->flush_issued) {3529 iwqp->flush_issued = 1;3530 issue_flush = 1;3531 }3532 } else if ((original_hw_tcp_state == IRDMA_TCP_STATE_CLOSE_WAIT) ||3533 ((original_ibqp_state == IB_QPS_RTS) &&3534 (last_ae == IRDMA_AE_LLP_CONNECTION_RESET))) {3535 issue_disconn = 1;3536 if (last_ae == IRDMA_AE_LLP_CONNECTION_RESET)3537 disconn_status = -ECONNRESET;3538 }3539 3540 if (original_hw_tcp_state == IRDMA_TCP_STATE_CLOSED ||3541 original_hw_tcp_state == IRDMA_TCP_STATE_TIME_WAIT ||3542 last_ae == IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE ||3543 last_ae == IRDMA_AE_BAD_CLOSE ||3544 last_ae == IRDMA_AE_LLP_CONNECTION_RESET || iwdev->rf->reset || !cm_id) {3545 issue_close = 1;3546 iwqp->cm_id = NULL;3547 qp->term_flags = 0;3548 if (!iwqp->flush_issued) {3549 iwqp->flush_issued = 1;3550 issue_flush = 1;3551 }3552 }3553 3554 spin_unlock_irqrestore(&iwqp->lock, flags);3555 if (issue_flush && !iwqp->sc_qp.qp_uk.destroy_pending) {3556 irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | IRDMA_FLUSH_RQ |3557 IRDMA_FLUSH_WAIT);3558 3559 if (qp->term_flags)3560 irdma_ib_qp_event(iwqp, qp->event_type);3561 }3562 3563 if (!cm_id || !cm_id->event_handler)3564 return;3565 3566 spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags);3567 if (!iwqp->cm_node) {3568 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags);3569 return;3570 }3571 refcount_inc(&iwqp->cm_node->refcnt);3572 3573 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags);3574 3575 if (issue_disconn) {3576 err = irdma_send_cm_event(iwqp->cm_node, cm_id,3577 IW_CM_EVENT_DISCONNECT,3578 disconn_status);3579 if (err)3580 ibdev_dbg(&iwdev->ibdev,3581 "CM: disconnect event failed: - cm_id = %p\n",3582 cm_id);3583 }3584 if (issue_close) {3585 cm_id->provider_data = iwqp;3586 err = irdma_send_cm_event(iwqp->cm_node, cm_id,3587 IW_CM_EVENT_CLOSE, 0);3588 if (err)3589 ibdev_dbg(&iwdev->ibdev,3590 "CM: close event failed: - cm_id = %p\n",3591 cm_id);3592 irdma_qp_disconnect(iwqp);3593 }3594 irdma_rem_ref_cm_node(iwqp->cm_node);3595}3596 3597/**3598 * irdma_disconnect_worker - worker for connection close3599 * @work: points or disconn structure3600 */3601static void irdma_disconnect_worker(struct work_struct *work)3602{3603 struct disconn_work *dwork = container_of(work, struct disconn_work, work);3604 struct irdma_qp *iwqp = dwork->iwqp;3605 3606 kfree(dwork);3607 irdma_cm_disconn_true(iwqp);3608 irdma_qp_rem_ref(&iwqp->ibqp);3609}3610 3611/**3612 * irdma_free_lsmm_rsrc - free lsmm memory and deregister3613 * @iwqp: associate qp for the connection3614 */3615void irdma_free_lsmm_rsrc(struct irdma_qp *iwqp)3616{3617 struct irdma_device *iwdev;3618 3619 iwdev = iwqp->iwdev;3620 3621 if (iwqp->ietf_mem.va) {3622 if (iwqp->lsmm_mr)3623 iwdev->ibdev.ops.dereg_mr(iwqp->lsmm_mr, NULL);3624 dma_free_coherent(iwdev->rf->sc_dev.hw->device,3625 iwqp->ietf_mem.size, iwqp->ietf_mem.va,3626 iwqp->ietf_mem.pa);3627 iwqp->ietf_mem.va = NULL;3628 }3629}3630 3631/**3632 * irdma_accept - registered call for connection to be accepted3633 * @cm_id: cm information for passive connection3634 * @conn_param: accept parameters3635 */3636int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)3637{3638 struct ib_qp *ibqp;3639 struct irdma_qp *iwqp;3640 struct irdma_device *iwdev;3641 struct irdma_sc_dev *dev;3642 struct irdma_cm_node *cm_node;3643 struct ib_qp_attr attr = {};3644 int passive_state;3645 struct ib_mr *ibmr;3646 struct irdma_pd *iwpd;3647 u16 buf_len = 0;3648 struct irdma_kmem_info accept;3649 u64 tagged_offset;3650 int wait_ret;3651 int ret = 0;3652 3653 ibqp = irdma_get_qp(cm_id->device, conn_param->qpn);3654 if (!ibqp)3655 return -EINVAL;3656 3657 iwqp = to_iwqp(ibqp);3658 iwdev = iwqp->iwdev;3659 dev = &iwdev->rf->sc_dev;3660 cm_node = cm_id->provider_data;3661 3662 if (((struct sockaddr_in *)&cm_id->local_addr)->sin_family == AF_INET) {3663 cm_node->ipv4 = true;3664 cm_node->vlan_id = irdma_get_vlan_ipv4(cm_node->loc_addr);3665 } else {3666 cm_node->ipv4 = false;3667 irdma_get_vlan_mac_ipv6(cm_node->loc_addr, &cm_node->vlan_id,3668 NULL);3669 }3670 ibdev_dbg(&iwdev->ibdev, "CM: Accept vlan_id=%d\n",3671 cm_node->vlan_id);3672 3673 trace_irdma_accept(cm_node, 0, NULL);3674 3675 if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) {3676 ret = -EINVAL;3677 goto error;3678 }3679 3680 passive_state = atomic_add_return(1, &cm_node->passive_state);3681 if (passive_state == IRDMA_SEND_RESET_EVENT) {3682 ret = -ECONNRESET;3683 goto error;3684 }3685 3686 buf_len = conn_param->private_data_len + IRDMA_MAX_IETF_SIZE;3687 iwqp->ietf_mem.size = ALIGN(buf_len, 1);3688 iwqp->ietf_mem.va = dma_alloc_coherent(dev->hw->device,3689 iwqp->ietf_mem.size,3690 &iwqp->ietf_mem.pa, GFP_KERNEL);3691 if (!iwqp->ietf_mem.va) {3692 ret = -ENOMEM;3693 goto error;3694 }3695 3696 cm_node->pdata.size = conn_param->private_data_len;3697 accept.addr = iwqp->ietf_mem.va;3698 accept.size = irdma_cm_build_mpa_frame(cm_node, &accept, MPA_KEY_REPLY);3699 memcpy((u8 *)accept.addr + accept.size, conn_param->private_data,3700 conn_param->private_data_len);3701 3702 if (cm_node->dev->ws_add(iwqp->sc_qp.vsi, cm_node->user_pri)) {3703 ret = -ENOMEM;3704 goto error;3705 }3706 iwqp->sc_qp.user_pri = cm_node->user_pri;3707 irdma_qp_add_qos(&iwqp->sc_qp);3708 /* setup our first outgoing iWarp send WQE (the IETF frame response) */3709 iwpd = iwqp->iwpd;3710 tagged_offset = (uintptr_t)iwqp->ietf_mem.va;3711 ibmr = irdma_reg_phys_mr(&iwpd->ibpd, iwqp->ietf_mem.pa, buf_len,3712 IB_ACCESS_LOCAL_WRITE, &tagged_offset);3713 if (IS_ERR(ibmr)) {3714 ret = -ENOMEM;3715 goto error;3716 }3717 3718 ibmr->pd = &iwpd->ibpd;3719 ibmr->device = iwpd->ibpd.device;3720 iwqp->lsmm_mr = ibmr;3721 if (iwqp->page)3722 iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(iwqp->page);3723 3724 cm_node->lsmm_size = accept.size + conn_param->private_data_len;3725 irdma_sc_send_lsmm(&iwqp->sc_qp, iwqp->ietf_mem.va, cm_node->lsmm_size,3726 ibmr->lkey);3727 3728 if (iwqp->page)3729 kunmap_local(iwqp->sc_qp.qp_uk.sq_base);3730 3731 iwqp->cm_id = cm_id;3732 cm_node->cm_id = cm_id;3733 3734 cm_id->provider_data = iwqp;3735 iwqp->active_conn = 0;3736 iwqp->cm_node = cm_node;3737 cm_node->iwqp = iwqp;3738 irdma_cm_init_tsa_conn(iwqp, cm_node);3739 irdma_qp_add_ref(&iwqp->ibqp);3740 cm_id->add_ref(cm_id);3741 3742 attr.qp_state = IB_QPS_RTS;3743 cm_node->qhash_set = false;3744 cm_node->cm_core->cm_free_ah(cm_node);3745 3746 irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);3747 if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) {3748 wait_ret = wait_event_interruptible_timeout(iwqp->waitq,3749 iwqp->rts_ae_rcvd,3750 IRDMA_MAX_TIMEOUT);3751 if (!wait_ret) {3752 ibdev_dbg(&iwdev->ibdev,3753 "CM: Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n",3754 cm_node, cm_node->loc_port,3755 cm_node->rem_port, cm_node->cm_id);3756 ret = -ECONNRESET;3757 goto error;3758 }3759 }3760 3761 irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_ESTABLISHED, 0);3762 cm_node->accelerated = true;3763 complete(&cm_node->establish_comp);3764 3765 if (cm_node->accept_pend) {3766 atomic_dec(&cm_node->listener->pend_accepts_cnt);3767 cm_node->accept_pend = 0;3768 }3769 3770 ibdev_dbg(&iwdev->ibdev,3771 "CM: rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n",3772 cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,3773 cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num);3774 cm_node->cm_core->stats_accepts++;3775 3776 return 0;3777error:3778 irdma_free_lsmm_rsrc(iwqp);3779 irdma_rem_ref_cm_node(cm_node);3780 3781 return ret;3782}3783 3784/**3785 * irdma_reject - registered call for connection to be rejected3786 * @cm_id: cm information for passive connection3787 * @pdata: private data to be sent3788 * @pdata_len: private data length3789 */3790int irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)3791{3792 struct irdma_device *iwdev;3793 struct irdma_cm_node *cm_node;3794 3795 cm_node = cm_id->provider_data;3796 cm_node->pdata.size = pdata_len;3797 3798 trace_irdma_reject(cm_node, 0, NULL);3799 3800 iwdev = to_iwdev(cm_id->device);3801 if (!iwdev)3802 return -EINVAL;3803 3804 cm_node->cm_core->stats_rejects++;3805 3806 if (pdata_len + sizeof(struct ietf_mpa_v2) > IRDMA_MAX_CM_BUF)3807 return -EINVAL;3808 3809 return irdma_cm_reject(cm_node, pdata, pdata_len);3810}3811 3812/**3813 * irdma_connect - registered call for connection to be established3814 * @cm_id: cm information for passive connection3815 * @conn_param: Information about the connection3816 */3817int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)3818{3819 struct ib_qp *ibqp;3820 struct irdma_qp *iwqp;3821 struct irdma_device *iwdev;3822 struct irdma_cm_node *cm_node;3823 struct irdma_cm_info cm_info;3824 struct sockaddr_in *laddr;3825 struct sockaddr_in *raddr;3826 struct sockaddr_in6 *laddr6;3827 struct sockaddr_in6 *raddr6;3828 int ret = 0;3829 3830 ibqp = irdma_get_qp(cm_id->device, conn_param->qpn);3831 if (!ibqp)3832 return -EINVAL;3833 iwqp = to_iwqp(ibqp);3834 if (!iwqp)3835 return -EINVAL;3836 iwdev = iwqp->iwdev;3837 if (!iwdev)3838 return -EINVAL;3839 3840 laddr = (struct sockaddr_in *)&cm_id->m_local_addr;3841 raddr = (struct sockaddr_in *)&cm_id->m_remote_addr;3842 laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;3843 raddr6 = (struct sockaddr_in6 *)&cm_id->m_remote_addr;3844 3845 if (!(laddr->sin_port) || !(raddr->sin_port))3846 return -EINVAL;3847 3848 iwqp->active_conn = 1;3849 iwqp->cm_id = NULL;3850 cm_id->provider_data = iwqp;3851 3852 /* set up the connection params for the node */3853 if (cm_id->remote_addr.ss_family == AF_INET) {3854 if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4)3855 return -EINVAL;3856 3857 cm_info.ipv4 = true;3858 memset(cm_info.loc_addr, 0, sizeof(cm_info.loc_addr));3859 memset(cm_info.rem_addr, 0, sizeof(cm_info.rem_addr));3860 cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);3861 cm_info.rem_addr[0] = ntohl(raddr->sin_addr.s_addr);3862 cm_info.loc_port = ntohs(laddr->sin_port);3863 cm_info.rem_port = ntohs(raddr->sin_port);3864 cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr);3865 } else {3866 if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6)3867 return -EINVAL;3868 3869 cm_info.ipv4 = false;3870 irdma_copy_ip_ntohl(cm_info.loc_addr,3871 laddr6->sin6_addr.in6_u.u6_addr32);3872 irdma_copy_ip_ntohl(cm_info.rem_addr,3873 raddr6->sin6_addr.in6_u.u6_addr32);3874 cm_info.loc_port = ntohs(laddr6->sin6_port);3875 cm_info.rem_port = ntohs(raddr6->sin6_port);3876 irdma_get_vlan_mac_ipv6(cm_info.loc_addr, &cm_info.vlan_id,3877 NULL);3878 }3879 cm_info.cm_id = cm_id;3880 cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;3881 cm_info.tos = cm_id->tos;3882 if (iwdev->vsi.dscp_mode) {3883 cm_info.user_pri =3884 iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(cm_info.tos)];3885 } else {3886 cm_info.user_pri = rt_tos2priority(cm_id->tos);3887 cm_info.user_pri = irdma_iw_get_vlan_prio(cm_info.loc_addr,3888 cm_info.user_pri,3889 cm_info.ipv4);3890 }3891 3892 if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, cm_info.user_pri))3893 return -ENOMEM;3894 iwqp->sc_qp.user_pri = cm_info.user_pri;3895 irdma_qp_add_qos(&iwqp->sc_qp);3896 ibdev_dbg(&iwdev->ibdev, "DCB: TOS:[%d] UP:[%d]\n", cm_id->tos,3897 cm_info.user_pri);3898 3899 trace_irdma_dcb_tos(iwdev, cm_id->tos, cm_info.user_pri);3900 3901 ret = irdma_create_cm_node(&iwdev->cm_core, iwdev, conn_param, &cm_info,3902 &cm_node);3903 if (ret)3904 return ret;3905 ret = cm_node->cm_core->cm_create_ah(cm_node, true);3906 if (ret)3907 goto err;3908 if (irdma_manage_qhash(iwdev, &cm_info,3909 IRDMA_QHASH_TYPE_TCP_ESTABLISHED,3910 IRDMA_QHASH_MANAGE_TYPE_ADD, NULL, true)) {3911 ret = -EINVAL;3912 goto err;3913 }3914 cm_node->qhash_set = true;3915 3916 cm_node->apbvt_entry = irdma_add_apbvt(iwdev, cm_info.loc_port);3917 if (!cm_node->apbvt_entry) {3918 ret = -EINVAL;3919 goto err;3920 }3921 3922 cm_node->apbvt_set = true;3923 iwqp->cm_node = cm_node;3924 cm_node->iwqp = iwqp;3925 iwqp->cm_id = cm_id;3926 irdma_qp_add_ref(&iwqp->ibqp);3927 cm_id->add_ref(cm_id);3928 3929 if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) {3930 cm_node->state = IRDMA_CM_STATE_SYN_SENT;3931 ret = irdma_send_syn(cm_node, 0);3932 if (ret)3933 goto err;3934 }3935 3936 ibdev_dbg(&iwdev->ibdev,3937 "CM: rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n",3938 cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,3939 cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num);3940 3941 trace_irdma_connect(cm_node, 0, NULL);3942 3943 return 0;3944 3945err:3946 if (cm_info.ipv4)3947 ibdev_dbg(&iwdev->ibdev,3948 "CM: connect() FAILED: dest addr=%pI4",3949 cm_info.rem_addr);3950 else3951 ibdev_dbg(&iwdev->ibdev,3952 "CM: connect() FAILED: dest addr=%pI6",3953 cm_info.rem_addr);3954 irdma_rem_ref_cm_node(cm_node);3955 iwdev->cm_core.stats_connect_errs++;3956 3957 return ret;3958}3959 3960/**3961 * irdma_create_listen - registered call creating listener3962 * @cm_id: cm information for passive connection3963 * @backlog: to max accept pending count3964 */3965int irdma_create_listen(struct iw_cm_id *cm_id, int backlog)3966{3967 struct irdma_device *iwdev;3968 struct irdma_cm_listener *cm_listen_node;3969 struct irdma_cm_info cm_info = {};3970 struct sockaddr_in *laddr;3971 struct sockaddr_in6 *laddr6;3972 bool wildcard = false;3973 int err;3974 3975 iwdev = to_iwdev(cm_id->device);3976 if (!iwdev)3977 return -EINVAL;3978 3979 laddr = (struct sockaddr_in *)&cm_id->m_local_addr;3980 laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;3981 cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;3982 3983 if (laddr->sin_family == AF_INET) {3984 if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4)3985 return -EINVAL;3986 3987 cm_info.ipv4 = true;3988 cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);3989 cm_info.loc_port = ntohs(laddr->sin_port);3990 3991 if (laddr->sin_addr.s_addr != htonl(INADDR_ANY)) {3992 cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr);3993 } else {3994 cm_info.vlan_id = 0xFFFF;3995 wildcard = true;3996 }3997 } else {3998 if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6)3999 return -EINVAL;4000 4001 cm_info.ipv4 = false;4002 irdma_copy_ip_ntohl(cm_info.loc_addr,4003 laddr6->sin6_addr.in6_u.u6_addr32);4004 cm_info.loc_port = ntohs(laddr6->sin6_port);4005 if (ipv6_addr_type(&laddr6->sin6_addr) != IPV6_ADDR_ANY) {4006 irdma_get_vlan_mac_ipv6(cm_info.loc_addr,4007 &cm_info.vlan_id, NULL);4008 } else {4009 cm_info.vlan_id = 0xFFFF;4010 wildcard = true;4011 }4012 }4013 4014 if (cm_info.vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)4015 cm_info.vlan_id = 0;4016 cm_info.backlog = backlog;4017 cm_info.cm_id = cm_id;4018 4019 trace_irdma_create_listen(iwdev, &cm_info);4020 4021 cm_listen_node = irdma_make_listen_node(&iwdev->cm_core, iwdev,4022 &cm_info);4023 if (!cm_listen_node) {4024 ibdev_dbg(&iwdev->ibdev,4025 "CM: cm_listen_node == NULL\n");4026 return -ENOMEM;4027 }4028 4029 cm_id->provider_data = cm_listen_node;4030 4031 cm_listen_node->tos = cm_id->tos;4032 if (iwdev->vsi.dscp_mode)4033 cm_listen_node->user_pri =4034 iwdev->vsi.dscp_map[irdma_tos2dscp(cm_id->tos)];4035 else4036 cm_listen_node->user_pri = rt_tos2priority(cm_id->tos);4037 cm_info.user_pri = cm_listen_node->user_pri;4038 if (!cm_listen_node->reused_node) {4039 if (wildcard) {4040 err = irdma_add_mqh(iwdev, &cm_info, cm_listen_node);4041 if (err)4042 goto error;4043 } else {4044 if (!iwdev->vsi.dscp_mode)4045 cm_listen_node->user_pri =4046 irdma_iw_get_vlan_prio(cm_info.loc_addr,4047 cm_info.user_pri,4048 cm_info.ipv4);4049 cm_info.user_pri = cm_listen_node->user_pri;4050 err = irdma_manage_qhash(iwdev, &cm_info,4051 IRDMA_QHASH_TYPE_TCP_SYN,4052 IRDMA_QHASH_MANAGE_TYPE_ADD,4053 NULL, true);4054 if (err)4055 goto error;4056 4057 cm_listen_node->qhash_set = true;4058 }4059 4060 cm_listen_node->apbvt_entry = irdma_add_apbvt(iwdev,4061 cm_info.loc_port);4062 if (!cm_listen_node->apbvt_entry)4063 goto error;4064 }4065 cm_id->add_ref(cm_id);4066 cm_listen_node->cm_core->stats_listen_created++;4067 ibdev_dbg(&iwdev->ibdev,4068 "CM: loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d\n",4069 cm_listen_node->loc_port, cm_listen_node->loc_addr,4070 cm_listen_node, cm_listen_node->cm_id,4071 cm_listen_node->qhash_set, cm_listen_node->vlan_id);4072 4073 return 0;4074 4075error:4076 4077 irdma_cm_del_listen(&iwdev->cm_core, cm_listen_node, false);4078 4079 return -EINVAL;4080}4081 4082/**4083 * irdma_destroy_listen - registered call to destroy listener4084 * @cm_id: cm information for passive connection4085 */4086int irdma_destroy_listen(struct iw_cm_id *cm_id)4087{4088 struct irdma_device *iwdev;4089 4090 iwdev = to_iwdev(cm_id->device);4091 if (cm_id->provider_data)4092 irdma_cm_del_listen(&iwdev->cm_core, cm_id->provider_data,4093 true);4094 else4095 ibdev_dbg(&iwdev->ibdev,4096 "CM: cm_id->provider_data was NULL\n");4097 4098 cm_id->rem_ref(cm_id);4099 4100 return 0;4101}4102 4103/**4104 * irdma_teardown_list_prep - add conn nodes slated for tear down to list4105 * @cm_core: cm's core4106 * @teardown_list: a list to which cm_node will be selected4107 * @ipaddr: pointer to ip address4108 * @nfo: pointer to cm_info structure instance4109 * @disconnect_all: flag indicating disconnect all QPs4110 */4111static void irdma_teardown_list_prep(struct irdma_cm_core *cm_core,4112 struct list_head *teardown_list,4113 u32 *ipaddr,4114 struct irdma_cm_info *nfo,4115 bool disconnect_all)4116{4117 struct irdma_cm_node *cm_node;4118 int bkt;4119 4120 hash_for_each_rcu(cm_core->cm_hash_tbl, bkt, cm_node, list) {4121 if ((disconnect_all ||4122 (nfo->vlan_id == cm_node->vlan_id &&4123 !memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16))) &&4124 refcount_inc_not_zero(&cm_node->refcnt))4125 list_add(&cm_node->teardown_entry, teardown_list);4126 }4127}4128 4129/**4130 * irdma_cm_event_connected - handle connected active node4131 * @event: the info for cm_node of connection4132 */4133static void irdma_cm_event_connected(struct irdma_cm_event *event)4134{4135 struct irdma_qp *iwqp;4136 struct irdma_device *iwdev;4137 struct irdma_cm_node *cm_node;4138 struct irdma_sc_dev *dev;4139 struct ib_qp_attr attr = {};4140 struct iw_cm_id *cm_id;4141 int status;4142 bool read0;4143 int wait_ret = 0;4144 4145 cm_node = event->cm_node;4146 cm_id = cm_node->cm_id;4147 iwqp = cm_id->provider_data;4148 iwdev = iwqp->iwdev;4149 dev = &iwdev->rf->sc_dev;4150 if (iwqp->sc_qp.qp_uk.destroy_pending) {4151 status = -ETIMEDOUT;4152 goto error;4153 }4154 4155 irdma_cm_init_tsa_conn(iwqp, cm_node);4156 read0 = (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO);4157 if (iwqp->page)4158 iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(iwqp->page);4159 irdma_sc_send_rtt(&iwqp->sc_qp, read0);4160 if (iwqp->page)4161 kunmap_local(iwqp->sc_qp.qp_uk.sq_base);4162 4163 attr.qp_state = IB_QPS_RTS;4164 cm_node->qhash_set = false;4165 irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);4166 if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) {4167 wait_ret = wait_event_interruptible_timeout(iwqp->waitq,4168 iwqp->rts_ae_rcvd,4169 IRDMA_MAX_TIMEOUT);4170 if (!wait_ret)4171 ibdev_dbg(&iwdev->ibdev,4172 "CM: Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n",4173 cm_node, cm_node->loc_port,4174 cm_node->rem_port, cm_node->cm_id);4175 }4176 4177 irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, 0);4178 cm_node->accelerated = true;4179 complete(&cm_node->establish_comp);4180 cm_node->cm_core->cm_free_ah(cm_node);4181 return;4182 4183error:4184 iwqp->cm_id = NULL;4185 cm_id->provider_data = NULL;4186 irdma_send_cm_event(event->cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY,4187 status);4188 irdma_rem_ref_cm_node(event->cm_node);4189}4190 4191/**4192 * irdma_cm_event_reset - handle reset4193 * @event: the info for cm_node of connection4194 */4195static void irdma_cm_event_reset(struct irdma_cm_event *event)4196{4197 struct irdma_cm_node *cm_node = event->cm_node;4198 struct iw_cm_id *cm_id = cm_node->cm_id;4199 struct irdma_qp *iwqp;4200 4201 if (!cm_id)4202 return;4203 4204 iwqp = cm_id->provider_data;4205 if (!iwqp)4206 return;4207 4208 ibdev_dbg(&cm_node->iwdev->ibdev,4209 "CM: reset event %p - cm_id = %p\n", event->cm_node, cm_id);4210 iwqp->cm_id = NULL;4211 4212 irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_DISCONNECT,4213 -ECONNRESET);4214 irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CLOSE, 0);4215}4216 4217/**4218 * irdma_cm_event_handler - send event to cm upper layer4219 * @work: pointer of cm event info.4220 */4221static void irdma_cm_event_handler(struct work_struct *work)4222{4223 struct irdma_cm_event *event = container_of(work, struct irdma_cm_event, event_work);4224 struct irdma_cm_node *cm_node;4225 4226 if (!event || !event->cm_node || !event->cm_node->cm_core)4227 return;4228 4229 cm_node = event->cm_node;4230 trace_irdma_cm_event_handler(cm_node, event->type, NULL);4231 4232 switch (event->type) {4233 case IRDMA_CM_EVENT_MPA_REQ:4234 irdma_send_cm_event(cm_node, cm_node->cm_id,4235 IW_CM_EVENT_CONNECT_REQUEST, 0);4236 break;4237 case IRDMA_CM_EVENT_RESET:4238 irdma_cm_event_reset(event);4239 break;4240 case IRDMA_CM_EVENT_CONNECTED:4241 if (!event->cm_node->cm_id ||4242 event->cm_node->state != IRDMA_CM_STATE_OFFLOADED)4243 break;4244 irdma_cm_event_connected(event);4245 break;4246 case IRDMA_CM_EVENT_MPA_REJECT:4247 if (!event->cm_node->cm_id ||4248 cm_node->state == IRDMA_CM_STATE_OFFLOADED)4249 break;4250 irdma_send_cm_event(cm_node, cm_node->cm_id,4251 IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED);4252 break;4253 case IRDMA_CM_EVENT_ABORTED:4254 if (!event->cm_node->cm_id ||4255 event->cm_node->state == IRDMA_CM_STATE_OFFLOADED)4256 break;4257 irdma_event_connect_error(event);4258 break;4259 default:4260 ibdev_dbg(&cm_node->iwdev->ibdev,4261 "CM: bad event type = %d\n", event->type);4262 break;4263 }4264 4265 irdma_rem_ref_cm_node(event->cm_node);4266 kfree(event);4267}4268 4269/**4270 * irdma_cm_post_event - queue event request for worker thread4271 * @event: cm node's info for up event call4272 */4273static void irdma_cm_post_event(struct irdma_cm_event *event)4274{4275 refcount_inc(&event->cm_node->refcnt);4276 INIT_WORK(&event->event_work, irdma_cm_event_handler);4277 queue_work(event->cm_node->cm_core->event_wq, &event->event_work);4278}4279 4280/**4281 * irdma_cm_teardown_connections - teardown QPs4282 * @iwdev: device pointer4283 * @ipaddr: Pointer to IPv4 or IPv6 address4284 * @nfo: Connection info4285 * @disconnect_all: flag indicating disconnect all QPs4286 *4287 * teardown QPs where source or destination addr matches ip addr4288 */4289void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr,4290 struct irdma_cm_info *nfo,4291 bool disconnect_all)4292{4293 struct irdma_cm_core *cm_core = &iwdev->cm_core;4294 struct list_head *list_core_temp;4295 struct list_head *list_node;4296 struct irdma_cm_node *cm_node;4297 struct list_head teardown_list;4298 struct ib_qp_attr attr;4299 4300 INIT_LIST_HEAD(&teardown_list);4301 4302 rcu_read_lock();4303 irdma_teardown_list_prep(cm_core, &teardown_list, ipaddr, nfo, disconnect_all);4304 rcu_read_unlock();4305 4306 list_for_each_safe (list_node, list_core_temp, &teardown_list) {4307 cm_node = container_of(list_node, struct irdma_cm_node,4308 teardown_entry);4309 attr.qp_state = IB_QPS_ERR;4310 irdma_modify_qp(&cm_node->iwqp->ibqp, &attr, IB_QP_STATE, NULL);4311 if (iwdev->rf->reset)4312 irdma_cm_disconn(cm_node->iwqp);4313 irdma_rem_ref_cm_node(cm_node);4314 }4315}4316 4317/**4318 * irdma_qhash_ctrl - enable/disable qhash for list4319 * @iwdev: device pointer4320 * @parent_listen_node: parent listen node4321 * @nfo: cm info node4322 * @ipaddr: Pointer to IPv4 or IPv6 address4323 * @ipv4: flag indicating IPv4 when true4324 * @ifup: flag indicating interface up when true4325 *4326 * Enables or disables the qhash for the node in the child4327 * listen list that matches ipaddr. If no matching IP was found4328 * it will allocate and add a new child listen node to the4329 * parent listen node. The listen_list_lock is assumed to be4330 * held when called.4331 */4332static void irdma_qhash_ctrl(struct irdma_device *iwdev,4333 struct irdma_cm_listener *parent_listen_node,4334 struct irdma_cm_info *nfo, u32 *ipaddr, bool ipv4,4335 bool ifup)4336{4337 struct list_head *child_listen_list = &parent_listen_node->child_listen_list;4338 struct irdma_cm_listener *child_listen_node;4339 struct list_head *pos, *tpos;4340 bool node_allocated = false;4341 enum irdma_quad_hash_manage_type op = ifup ?4342 IRDMA_QHASH_MANAGE_TYPE_ADD :4343 IRDMA_QHASH_MANAGE_TYPE_DELETE;4344 int err;4345 4346 list_for_each_safe (pos, tpos, child_listen_list) {4347 child_listen_node = list_entry(pos, struct irdma_cm_listener,4348 child_listen_list);4349 if (!memcmp(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16))4350 goto set_qhash;4351 }4352 4353 /* if not found then add a child listener if interface is going up */4354 if (!ifup)4355 return;4356 child_listen_node = kmemdup(parent_listen_node,4357 sizeof(*child_listen_node), GFP_ATOMIC);4358 if (!child_listen_node)4359 return;4360 4361 node_allocated = true;4362 memcpy(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16);4363 4364set_qhash:4365 memcpy(nfo->loc_addr, child_listen_node->loc_addr,4366 sizeof(nfo->loc_addr));4367 nfo->vlan_id = child_listen_node->vlan_id;4368 err = irdma_manage_qhash(iwdev, nfo, IRDMA_QHASH_TYPE_TCP_SYN, op, NULL,4369 false);4370 if (!err) {4371 child_listen_node->qhash_set = ifup;4372 if (node_allocated)4373 list_add(&child_listen_node->child_listen_list,4374 &parent_listen_node->child_listen_list);4375 } else if (node_allocated) {4376 kfree(child_listen_node);4377 }4378}4379 4380/**4381 * irdma_if_notify - process an ifdown on an interface4382 * @iwdev: device pointer4383 * @netdev: network device structure4384 * @ipaddr: Pointer to IPv4 or IPv6 address4385 * @ipv4: flag indicating IPv4 when true4386 * @ifup: flag indicating interface up when true4387 */4388void irdma_if_notify(struct irdma_device *iwdev, struct net_device *netdev,4389 u32 *ipaddr, bool ipv4, bool ifup)4390{4391 struct irdma_cm_core *cm_core = &iwdev->cm_core;4392 unsigned long flags;4393 struct irdma_cm_listener *listen_node;4394 static const u32 ip_zero[4] = { 0, 0, 0, 0 };4395 struct irdma_cm_info nfo = {};4396 u16 vlan_id = rdma_vlan_dev_vlan_id(netdev);4397 enum irdma_quad_hash_manage_type op = ifup ?4398 IRDMA_QHASH_MANAGE_TYPE_ADD :4399 IRDMA_QHASH_MANAGE_TYPE_DELETE;4400 4401 nfo.vlan_id = vlan_id;4402 nfo.ipv4 = ipv4;4403 nfo.qh_qpid = 1;4404 4405 /* Disable or enable qhash for listeners */4406 spin_lock_irqsave(&cm_core->listen_list_lock, flags);4407 list_for_each_entry (listen_node, &cm_core->listen_list, list) {4408 if (vlan_id != listen_node->vlan_id ||4409 (memcmp(listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16) &&4410 memcmp(listen_node->loc_addr, ip_zero, ipv4 ? 4 : 16)))4411 continue;4412 4413 memcpy(nfo.loc_addr, listen_node->loc_addr,4414 sizeof(nfo.loc_addr));4415 nfo.loc_port = listen_node->loc_port;4416 nfo.user_pri = listen_node->user_pri;4417 if (!list_empty(&listen_node->child_listen_list)) {4418 irdma_qhash_ctrl(iwdev, listen_node, &nfo, ipaddr, ipv4,4419 ifup);4420 } else if (memcmp(listen_node->loc_addr, ip_zero,4421 ipv4 ? 4 : 16)) {4422 if (!irdma_manage_qhash(iwdev, &nfo,4423 IRDMA_QHASH_TYPE_TCP_SYN, op,4424 NULL, false))4425 listen_node->qhash_set = ifup;4426 }4427 }4428 spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);4429 4430 /* disconnect any connected qp's on ifdown */4431 if (!ifup)4432 irdma_cm_teardown_connections(iwdev, ipaddr, &nfo, false);4433}4434