2019 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2016 Chelsio Communications, Inc.4 */5 6#include <linux/module.h>7#include <linux/list.h>8#include <linux/workqueue.h>9#include <linux/skbuff.h>10#include <linux/timer.h>11#include <linux/notifier.h>12#include <linux/inetdevice.h>13#include <linux/ip.h>14#include <linux/tcp.h>15#include <linux/if_vlan.h>16 17#include <net/neighbour.h>18#include <net/netevent.h>19#include <net/route.h>20#include <net/tcp.h>21#include <net/ip6_route.h>22#include <net/addrconf.h>23 24#include <libcxgb_cm.h>25#include "cxgbit.h"26#include "clip_tbl.h"27 28static void cxgbit_init_wr_wait(struct cxgbit_wr_wait *wr_waitp)29{30 wr_waitp->ret = 0;31 reinit_completion(&wr_waitp->completion);32}33 34static void35cxgbit_wake_up(struct cxgbit_wr_wait *wr_waitp, const char *func, u8 ret)36{37 if (ret == CPL_ERR_NONE)38 wr_waitp->ret = 0;39 else40 wr_waitp->ret = -EIO;41 42 if (wr_waitp->ret)43 pr_err("%s: err:%u", func, ret);44 45 complete(&wr_waitp->completion);46}47 48static int49cxgbit_wait_for_reply(struct cxgbit_device *cdev,50 struct cxgbit_wr_wait *wr_waitp, u32 tid, u32 timeout,51 const char *func)52{53 int ret;54 55 if (!test_bit(CDEV_STATE_UP, &cdev->flags)) {56 wr_waitp->ret = -EIO;57 goto out;58 }59 60 ret = wait_for_completion_timeout(&wr_waitp->completion, timeout * HZ);61 if (!ret) {62 pr_info("%s - Device %s not responding tid %u\n",63 func, pci_name(cdev->lldi.pdev), tid);64 wr_waitp->ret = -ETIMEDOUT;65 }66out:67 if (wr_waitp->ret)68 pr_info("%s: FW reply %d tid %u\n",69 pci_name(cdev->lldi.pdev), wr_waitp->ret, tid);70 return wr_waitp->ret;71}72 73static int cxgbit_np_hashfn(const struct cxgbit_np *cnp)74{75 return ((unsigned long)cnp >> 10) & (NP_INFO_HASH_SIZE - 1);76}77 78static struct np_info *79cxgbit_np_hash_add(struct cxgbit_device *cdev, struct cxgbit_np *cnp,80 unsigned int stid)81{82 struct np_info *p = kzalloc(sizeof(*p), GFP_KERNEL);83 84 if (p) {85 int bucket = cxgbit_np_hashfn(cnp);86 87 p->cnp = cnp;88 p->stid = stid;89 spin_lock(&cdev->np_lock);90 p->next = cdev->np_hash_tab[bucket];91 cdev->np_hash_tab[bucket] = p;92 spin_unlock(&cdev->np_lock);93 }94 95 return p;96}97 98static int99cxgbit_np_hash_find(struct cxgbit_device *cdev, struct cxgbit_np *cnp)100{101 int stid = -1, bucket = cxgbit_np_hashfn(cnp);102 struct np_info *p;103 104 spin_lock(&cdev->np_lock);105 for (p = cdev->np_hash_tab[bucket]; p; p = p->next) {106 if (p->cnp == cnp) {107 stid = p->stid;108 break;109 }110 }111 spin_unlock(&cdev->np_lock);112 113 return stid;114}115 116static int cxgbit_np_hash_del(struct cxgbit_device *cdev, struct cxgbit_np *cnp)117{118 int stid = -1, bucket = cxgbit_np_hashfn(cnp);119 struct np_info *p, **prev = &cdev->np_hash_tab[bucket];120 121 spin_lock(&cdev->np_lock);122 for (p = *prev; p; prev = &p->next, p = p->next) {123 if (p->cnp == cnp) {124 stid = p->stid;125 *prev = p->next;126 kfree(p);127 break;128 }129 }130 spin_unlock(&cdev->np_lock);131 132 return stid;133}134 135void _cxgbit_free_cnp(struct kref *kref)136{137 struct cxgbit_np *cnp;138 139 cnp = container_of(kref, struct cxgbit_np, kref);140 kfree(cnp);141}142 143static int144cxgbit_create_server6(struct cxgbit_device *cdev, unsigned int stid,145 struct cxgbit_np *cnp)146{147 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)148 &cnp->com.local_addr;149 int addr_type;150 int ret;151 152 pr_debug("%s: dev = %s; stid = %u; sin6_port = %u\n",153 __func__, cdev->lldi.ports[0]->name, stid, sin6->sin6_port);154 155 addr_type = ipv6_addr_type((const struct in6_addr *)156 &sin6->sin6_addr);157 if (addr_type != IPV6_ADDR_ANY) {158 ret = cxgb4_clip_get(cdev->lldi.ports[0],159 (const u32 *)&sin6->sin6_addr.s6_addr, 1);160 if (ret) {161 pr_err("Unable to find clip table entry. laddr %pI6. Error:%d.\n",162 sin6->sin6_addr.s6_addr, ret);163 return -ENOMEM;164 }165 }166 167 cxgbit_get_cnp(cnp);168 cxgbit_init_wr_wait(&cnp->com.wr_wait);169 170 ret = cxgb4_create_server6(cdev->lldi.ports[0],171 stid, &sin6->sin6_addr,172 sin6->sin6_port,173 cdev->lldi.rxq_ids[0]);174 if (!ret)175 ret = cxgbit_wait_for_reply(cdev, &cnp->com.wr_wait,176 0, 10, __func__);177 else if (ret > 0)178 ret = net_xmit_errno(ret);179 else180 cxgbit_put_cnp(cnp);181 182 if (ret) {183 if (ret != -ETIMEDOUT)184 cxgb4_clip_release(cdev->lldi.ports[0],185 (const u32 *)&sin6->sin6_addr.s6_addr, 1);186 187 pr_err("create server6 err %d stid %d laddr %pI6 lport %d\n",188 ret, stid, sin6->sin6_addr.s6_addr,189 ntohs(sin6->sin6_port));190 }191 192 return ret;193}194 195static int196cxgbit_create_server4(struct cxgbit_device *cdev, unsigned int stid,197 struct cxgbit_np *cnp)198{199 struct sockaddr_in *sin = (struct sockaddr_in *)200 &cnp->com.local_addr;201 int ret;202 203 pr_debug("%s: dev = %s; stid = %u; sin_port = %u\n",204 __func__, cdev->lldi.ports[0]->name, stid, sin->sin_port);205 206 cxgbit_get_cnp(cnp);207 cxgbit_init_wr_wait(&cnp->com.wr_wait);208 209 ret = cxgb4_create_server(cdev->lldi.ports[0],210 stid, sin->sin_addr.s_addr,211 sin->sin_port, 0,212 cdev->lldi.rxq_ids[0]);213 if (!ret)214 ret = cxgbit_wait_for_reply(cdev,215 &cnp->com.wr_wait,216 0, 10, __func__);217 else if (ret > 0)218 ret = net_xmit_errno(ret);219 else220 cxgbit_put_cnp(cnp);221 222 if (ret)223 pr_err("create server failed err %d stid %d laddr %pI4 lport %d\n",224 ret, stid, &sin->sin_addr, ntohs(sin->sin_port));225 return ret;226}227 228struct cxgbit_device *cxgbit_find_device(struct net_device *ndev, u8 *port_id)229{230 struct cxgbit_device *cdev;231 u8 i;232 233 list_for_each_entry(cdev, &cdev_list_head, list) {234 struct cxgb4_lld_info *lldi = &cdev->lldi;235 236 for (i = 0; i < lldi->nports; i++) {237 if (lldi->ports[i] == ndev) {238 if (port_id)239 *port_id = i;240 return cdev;241 }242 }243 }244 245 return NULL;246}247 248static struct net_device *cxgbit_get_real_dev(struct net_device *ndev)249{250 if (ndev->priv_flags & IFF_BONDING) {251 pr_err("Bond devices are not supported. Interface:%s\n",252 ndev->name);253 return NULL;254 }255 256 if (is_vlan_dev(ndev))257 return vlan_dev_real_dev(ndev);258 259 return ndev;260}261 262static struct net_device *cxgbit_ipv4_netdev(__be32 saddr)263{264 struct net_device *ndev;265 266 ndev = __ip_dev_find(&init_net, saddr, false);267 if (!ndev)268 return NULL;269 270 return cxgbit_get_real_dev(ndev);271}272 273static struct net_device *cxgbit_ipv6_netdev(struct in6_addr *addr6)274{275 struct net_device *ndev = NULL;276 bool found = false;277 278 if (IS_ENABLED(CONFIG_IPV6)) {279 for_each_netdev_rcu(&init_net, ndev)280 if (ipv6_chk_addr(&init_net, addr6, ndev, 1)) {281 found = true;282 break;283 }284 }285 if (!found)286 return NULL;287 return cxgbit_get_real_dev(ndev);288}289 290static struct cxgbit_device *cxgbit_find_np_cdev(struct cxgbit_np *cnp)291{292 struct sockaddr_storage *sockaddr = &cnp->com.local_addr;293 int ss_family = sockaddr->ss_family;294 struct net_device *ndev = NULL;295 struct cxgbit_device *cdev = NULL;296 297 rcu_read_lock();298 if (ss_family == AF_INET) {299 struct sockaddr_in *sin;300 301 sin = (struct sockaddr_in *)sockaddr;302 ndev = cxgbit_ipv4_netdev(sin->sin_addr.s_addr);303 } else if (ss_family == AF_INET6) {304 struct sockaddr_in6 *sin6;305 306 sin6 = (struct sockaddr_in6 *)sockaddr;307 ndev = cxgbit_ipv6_netdev(&sin6->sin6_addr);308 }309 if (!ndev)310 goto out;311 312 cdev = cxgbit_find_device(ndev, NULL);313out:314 rcu_read_unlock();315 return cdev;316}317 318static bool cxgbit_inaddr_any(struct cxgbit_np *cnp)319{320 struct sockaddr_storage *sockaddr = &cnp->com.local_addr;321 int ss_family = sockaddr->ss_family;322 int addr_type;323 324 if (ss_family == AF_INET) {325 struct sockaddr_in *sin;326 327 sin = (struct sockaddr_in *)sockaddr;328 if (sin->sin_addr.s_addr == htonl(INADDR_ANY))329 return true;330 } else if (ss_family == AF_INET6) {331 struct sockaddr_in6 *sin6;332 333 sin6 = (struct sockaddr_in6 *)sockaddr;334 addr_type = ipv6_addr_type((const struct in6_addr *)335 &sin6->sin6_addr);336 if (addr_type == IPV6_ADDR_ANY)337 return true;338 }339 return false;340}341 342static int343__cxgbit_setup_cdev_np(struct cxgbit_device *cdev, struct cxgbit_np *cnp)344{345 int stid, ret;346 int ss_family = cnp->com.local_addr.ss_family;347 348 if (!test_bit(CDEV_STATE_UP, &cdev->flags))349 return -EINVAL;350 351 stid = cxgb4_alloc_stid(cdev->lldi.tids, ss_family, cnp);352 if (stid < 0)353 return -EINVAL;354 355 if (!cxgbit_np_hash_add(cdev, cnp, stid)) {356 cxgb4_free_stid(cdev->lldi.tids, stid, ss_family);357 return -EINVAL;358 }359 360 if (ss_family == AF_INET)361 ret = cxgbit_create_server4(cdev, stid, cnp);362 else363 ret = cxgbit_create_server6(cdev, stid, cnp);364 365 if (ret) {366 if (ret != -ETIMEDOUT)367 cxgb4_free_stid(cdev->lldi.tids, stid,368 ss_family);369 cxgbit_np_hash_del(cdev, cnp);370 return ret;371 }372 return ret;373}374 375static int cxgbit_setup_cdev_np(struct cxgbit_np *cnp)376{377 struct cxgbit_device *cdev;378 int ret = -1;379 380 mutex_lock(&cdev_list_lock);381 cdev = cxgbit_find_np_cdev(cnp);382 if (!cdev)383 goto out;384 385 if (cxgbit_np_hash_find(cdev, cnp) >= 0)386 goto out;387 388 if (__cxgbit_setup_cdev_np(cdev, cnp))389 goto out;390 391 cnp->com.cdev = cdev;392 ret = 0;393out:394 mutex_unlock(&cdev_list_lock);395 return ret;396}397 398static int cxgbit_setup_all_np(struct cxgbit_np *cnp)399{400 struct cxgbit_device *cdev;401 int ret;402 u32 count = 0;403 404 mutex_lock(&cdev_list_lock);405 list_for_each_entry(cdev, &cdev_list_head, list) {406 if (cxgbit_np_hash_find(cdev, cnp) >= 0) {407 mutex_unlock(&cdev_list_lock);408 return -1;409 }410 }411 412 list_for_each_entry(cdev, &cdev_list_head, list) {413 ret = __cxgbit_setup_cdev_np(cdev, cnp);414 if (ret == -ETIMEDOUT)415 break;416 if (ret != 0)417 continue;418 count++;419 }420 mutex_unlock(&cdev_list_lock);421 422 return count ? 0 : -1;423}424 425int cxgbit_setup_np(struct iscsi_np *np, struct sockaddr_storage *ksockaddr)426{427 struct cxgbit_np *cnp;428 int ret;429 430 if ((ksockaddr->ss_family != AF_INET) &&431 (ksockaddr->ss_family != AF_INET6))432 return -EINVAL;433 434 cnp = kzalloc(sizeof(*cnp), GFP_KERNEL);435 if (!cnp)436 return -ENOMEM;437 438 init_waitqueue_head(&cnp->accept_wait);439 init_completion(&cnp->com.wr_wait.completion);440 init_completion(&cnp->accept_comp);441 INIT_LIST_HEAD(&cnp->np_accept_list);442 spin_lock_init(&cnp->np_accept_lock);443 kref_init(&cnp->kref);444 memcpy(&np->np_sockaddr, ksockaddr,445 sizeof(struct sockaddr_storage));446 memcpy(&cnp->com.local_addr, &np->np_sockaddr,447 sizeof(cnp->com.local_addr));448 449 cnp->np = np;450 cnp->com.cdev = NULL;451 452 if (cxgbit_inaddr_any(cnp))453 ret = cxgbit_setup_all_np(cnp);454 else455 ret = cxgbit_setup_cdev_np(cnp);456 457 if (ret) {458 cxgbit_put_cnp(cnp);459 return -EINVAL;460 }461 462 np->np_context = cnp;463 cnp->com.state = CSK_STATE_LISTEN;464 return 0;465}466 467static void468cxgbit_set_conn_info(struct iscsi_np *np, struct iscsit_conn *conn,469 struct cxgbit_sock *csk)470{471 conn->login_family = np->np_sockaddr.ss_family;472 conn->login_sockaddr = csk->com.remote_addr;473 conn->local_sockaddr = csk->com.local_addr;474}475 476int cxgbit_accept_np(struct iscsi_np *np, struct iscsit_conn *conn)477{478 struct cxgbit_np *cnp = np->np_context;479 struct cxgbit_sock *csk;480 int ret = 0;481 482accept_wait:483 ret = wait_for_completion_interruptible(&cnp->accept_comp);484 if (ret)485 return -ENODEV;486 487 spin_lock_bh(&np->np_thread_lock);488 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) {489 spin_unlock_bh(&np->np_thread_lock);490 /**491 * No point in stalling here when np_thread492 * is in state RESET/SHUTDOWN/EXIT - bail493 **/494 return -ENODEV;495 }496 spin_unlock_bh(&np->np_thread_lock);497 498 spin_lock_bh(&cnp->np_accept_lock);499 if (list_empty(&cnp->np_accept_list)) {500 spin_unlock_bh(&cnp->np_accept_lock);501 goto accept_wait;502 }503 504 csk = list_first_entry(&cnp->np_accept_list,505 struct cxgbit_sock,506 accept_node);507 508 list_del_init(&csk->accept_node);509 spin_unlock_bh(&cnp->np_accept_lock);510 conn->context = csk;511 csk->conn = conn;512 513 cxgbit_set_conn_info(np, conn, csk);514 return 0;515}516 517static int518__cxgbit_free_cdev_np(struct cxgbit_device *cdev, struct cxgbit_np *cnp)519{520 int stid, ret;521 bool ipv6 = false;522 523 stid = cxgbit_np_hash_del(cdev, cnp);524 if (stid < 0)525 return -EINVAL;526 if (!test_bit(CDEV_STATE_UP, &cdev->flags))527 return -EINVAL;528 529 if (cnp->np->np_sockaddr.ss_family == AF_INET6)530 ipv6 = true;531 532 cxgbit_get_cnp(cnp);533 cxgbit_init_wr_wait(&cnp->com.wr_wait);534 ret = cxgb4_remove_server(cdev->lldi.ports[0], stid,535 cdev->lldi.rxq_ids[0], ipv6);536 537 if (ret > 0)538 ret = net_xmit_errno(ret);539 540 if (ret) {541 cxgbit_put_cnp(cnp);542 return ret;543 }544 545 ret = cxgbit_wait_for_reply(cdev, &cnp->com.wr_wait,546 0, 10, __func__);547 if (ret == -ETIMEDOUT)548 return ret;549 550 if (ipv6 && cnp->com.cdev) {551 struct sockaddr_in6 *sin6;552 553 sin6 = (struct sockaddr_in6 *)&cnp->com.local_addr;554 cxgb4_clip_release(cdev->lldi.ports[0],555 (const u32 *)&sin6->sin6_addr.s6_addr,556 1);557 }558 559 cxgb4_free_stid(cdev->lldi.tids, stid,560 cnp->com.local_addr.ss_family);561 return 0;562}563 564static void cxgbit_free_all_np(struct cxgbit_np *cnp)565{566 struct cxgbit_device *cdev;567 int ret;568 569 mutex_lock(&cdev_list_lock);570 list_for_each_entry(cdev, &cdev_list_head, list) {571 ret = __cxgbit_free_cdev_np(cdev, cnp);572 if (ret == -ETIMEDOUT)573 break;574 }575 mutex_unlock(&cdev_list_lock);576}577 578static void cxgbit_free_cdev_np(struct cxgbit_np *cnp)579{580 struct cxgbit_device *cdev;581 bool found = false;582 583 mutex_lock(&cdev_list_lock);584 list_for_each_entry(cdev, &cdev_list_head, list) {585 if (cdev == cnp->com.cdev) {586 found = true;587 break;588 }589 }590 if (!found)591 goto out;592 593 __cxgbit_free_cdev_np(cdev, cnp);594out:595 mutex_unlock(&cdev_list_lock);596}597 598static void __cxgbit_free_conn(struct cxgbit_sock *csk);599 600void cxgbit_free_np(struct iscsi_np *np)601{602 struct cxgbit_np *cnp = np->np_context;603 struct cxgbit_sock *csk, *tmp;604 605 cnp->com.state = CSK_STATE_DEAD;606 if (cnp->com.cdev)607 cxgbit_free_cdev_np(cnp);608 else609 cxgbit_free_all_np(cnp);610 611 spin_lock_bh(&cnp->np_accept_lock);612 list_for_each_entry_safe(csk, tmp, &cnp->np_accept_list, accept_node) {613 list_del_init(&csk->accept_node);614 __cxgbit_free_conn(csk);615 }616 spin_unlock_bh(&cnp->np_accept_lock);617 618 np->np_context = NULL;619 cxgbit_put_cnp(cnp);620}621 622static void cxgbit_send_halfclose(struct cxgbit_sock *csk)623{624 struct sk_buff *skb;625 u32 len = roundup(sizeof(struct cpl_close_con_req), 16);626 627 skb = alloc_skb(len, GFP_ATOMIC);628 if (!skb)629 return;630 631 cxgb_mk_close_con_req(skb, len, csk->tid, csk->txq_idx,632 NULL, NULL);633 634 cxgbit_skcb_flags(skb) |= SKCBF_TX_FLAG_COMPL;635 __skb_queue_tail(&csk->txq, skb);636 cxgbit_push_tx_frames(csk);637}638 639static void cxgbit_arp_failure_discard(void *handle, struct sk_buff *skb)640{641 struct cxgbit_sock *csk = handle;642 643 pr_debug("%s cxgbit_device %p\n", __func__, handle);644 kfree_skb(skb);645 cxgbit_put_csk(csk);646}647 648static void cxgbit_abort_arp_failure(void *handle, struct sk_buff *skb)649{650 struct cxgbit_device *cdev = handle;651 struct cpl_abort_req *req = cplhdr(skb);652 653 pr_debug("%s cdev %p\n", __func__, cdev);654 req->cmd = CPL_ABORT_NO_RST;655 cxgbit_ofld_send(cdev, skb);656}657 658static int cxgbit_send_abort_req(struct cxgbit_sock *csk)659{660 struct sk_buff *skb;661 u32 len = roundup(sizeof(struct cpl_abort_req), 16);662 663 pr_debug("%s: csk %p tid %u; state %d\n",664 __func__, csk, csk->tid, csk->com.state);665 666 __skb_queue_purge(&csk->txq);667 668 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags))669 cxgbit_send_tx_flowc_wr(csk);670 671 skb = __skb_dequeue(&csk->skbq);672 cxgb_mk_abort_req(skb, len, csk->tid, csk->txq_idx,673 csk->com.cdev, cxgbit_abort_arp_failure);674 675 return cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);676}677 678static void679__cxgbit_abort_conn(struct cxgbit_sock *csk, struct sk_buff *skb)680{681 __kfree_skb(skb);682 683 if (csk->com.state != CSK_STATE_ESTABLISHED)684 goto no_abort;685 686 set_bit(CSK_ABORT_RPL_WAIT, &csk->com.flags);687 csk->com.state = CSK_STATE_ABORTING;688 689 cxgbit_send_abort_req(csk);690 691 return;692 693no_abort:694 cxgbit_wake_up(&csk->com.wr_wait, __func__, CPL_ERR_NONE);695 cxgbit_put_csk(csk);696}697 698void cxgbit_abort_conn(struct cxgbit_sock *csk)699{700 struct sk_buff *skb = alloc_skb(0, GFP_KERNEL | __GFP_NOFAIL);701 702 cxgbit_get_csk(csk);703 cxgbit_init_wr_wait(&csk->com.wr_wait);704 705 spin_lock_bh(&csk->lock);706 if (csk->lock_owner) {707 cxgbit_skcb_rx_backlog_fn(skb) = __cxgbit_abort_conn;708 __skb_queue_tail(&csk->backlogq, skb);709 } else {710 __cxgbit_abort_conn(csk, skb);711 }712 spin_unlock_bh(&csk->lock);713 714 cxgbit_wait_for_reply(csk->com.cdev, &csk->com.wr_wait,715 csk->tid, 600, __func__);716}717 718static void __cxgbit_free_conn(struct cxgbit_sock *csk)719{720 struct iscsit_conn *conn = csk->conn;721 bool release = false;722 723 pr_debug("%s: state %d\n",724 __func__, csk->com.state);725 726 spin_lock_bh(&csk->lock);727 switch (csk->com.state) {728 case CSK_STATE_ESTABLISHED:729 if (conn && (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)) {730 csk->com.state = CSK_STATE_CLOSING;731 cxgbit_send_halfclose(csk);732 } else {733 csk->com.state = CSK_STATE_ABORTING;734 cxgbit_send_abort_req(csk);735 }736 break;737 case CSK_STATE_CLOSING:738 csk->com.state = CSK_STATE_MORIBUND;739 cxgbit_send_halfclose(csk);740 break;741 case CSK_STATE_DEAD:742 release = true;743 break;744 default:745 pr_err("%s: csk %p; state %d\n",746 __func__, csk, csk->com.state);747 }748 spin_unlock_bh(&csk->lock);749 750 if (release)751 cxgbit_put_csk(csk);752}753 754void cxgbit_free_conn(struct iscsit_conn *conn)755{756 __cxgbit_free_conn(conn->context);757}758 759static void cxgbit_set_emss(struct cxgbit_sock *csk, u16 opt)760{761 csk->emss = csk->com.cdev->lldi.mtus[TCPOPT_MSS_G(opt)] -762 ((csk->com.remote_addr.ss_family == AF_INET) ?763 sizeof(struct iphdr) : sizeof(struct ipv6hdr)) -764 sizeof(struct tcphdr);765 csk->mss = csk->emss;766 if (TCPOPT_TSTAMP_G(opt))767 csk->emss -= round_up(TCPOLEN_TIMESTAMP, 4);768 if (csk->emss < 128)769 csk->emss = 128;770 if (csk->emss & 7)771 pr_info("Warning: misaligned mtu idx %u mss %u emss=%u\n",772 TCPOPT_MSS_G(opt), csk->mss, csk->emss);773 pr_debug("%s mss_idx %u mss %u emss=%u\n", __func__, TCPOPT_MSS_G(opt),774 csk->mss, csk->emss);775}776 777static void cxgbit_free_skb(struct cxgbit_sock *csk)778{779 struct sk_buff *skb;780 781 __skb_queue_purge(&csk->txq);782 __skb_queue_purge(&csk->rxq);783 __skb_queue_purge(&csk->backlogq);784 __skb_queue_purge(&csk->ppodq);785 __skb_queue_purge(&csk->skbq);786 787 while ((skb = cxgbit_sock_dequeue_wr(csk)))788 kfree_skb(skb);789 790 __kfree_skb(csk->lro_hskb);791}792 793void _cxgbit_free_csk(struct kref *kref)794{795 struct cxgbit_sock *csk;796 struct cxgbit_device *cdev;797 798 csk = container_of(kref, struct cxgbit_sock, kref);799 800 pr_debug("%s csk %p state %d\n", __func__, csk, csk->com.state);801 802 if (csk->com.local_addr.ss_family == AF_INET6) {803 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)804 &csk->com.local_addr;805 cxgb4_clip_release(csk->com.cdev->lldi.ports[0],806 (const u32 *)807 &sin6->sin6_addr.s6_addr, 1);808 }809 810 cxgb4_remove_tid(csk->com.cdev->lldi.tids, 0, csk->tid,811 csk->com.local_addr.ss_family);812 dst_release(csk->dst);813 cxgb4_l2t_release(csk->l2t);814 815 cdev = csk->com.cdev;816 spin_lock_bh(&cdev->cskq.lock);817 list_del(&csk->list);818 spin_unlock_bh(&cdev->cskq.lock);819 820 cxgbit_free_skb(csk);821 cxgbit_put_cnp(csk->cnp);822 cxgbit_put_cdev(cdev);823 824 kfree(csk);825}826 827static void cxgbit_set_tcp_window(struct cxgbit_sock *csk, struct port_info *pi)828{829 unsigned int linkspeed;830 u8 scale;831 832 linkspeed = pi->link_cfg.speed;833 scale = linkspeed / SPEED_10000;834 835#define CXGBIT_10G_RCV_WIN (256 * 1024)836 csk->rcv_win = CXGBIT_10G_RCV_WIN;837 if (scale)838 csk->rcv_win *= scale;839 csk->rcv_win = min(csk->rcv_win, RCV_BUFSIZ_M << 10);840 841#define CXGBIT_10G_SND_WIN (256 * 1024)842 csk->snd_win = CXGBIT_10G_SND_WIN;843 if (scale)844 csk->snd_win *= scale;845 csk->snd_win = min(csk->snd_win, 512U * 1024);846 847 pr_debug("%s snd_win %d rcv_win %d\n",848 __func__, csk->snd_win, csk->rcv_win);849}850 851#ifdef CONFIG_CHELSIO_T4_DCB852static u8 cxgbit_get_iscsi_dcb_state(struct net_device *ndev)853{854 return ndev->dcbnl_ops->getstate(ndev);855}856 857static int cxgbit_select_priority(int pri_mask)858{859 if (!pri_mask)860 return 0;861 862 return (ffs(pri_mask) - 1);863}864 865static u8 cxgbit_get_iscsi_dcb_priority(struct net_device *ndev, u16 local_port)866{867 int ret;868 u8 caps;869 870 struct dcb_app iscsi_dcb_app = {871 .protocol = local_port872 };873 874 ret = (int)ndev->dcbnl_ops->getcap(ndev, DCB_CAP_ATTR_DCBX, &caps);875 876 if (ret)877 return 0;878 879 if (caps & DCB_CAP_DCBX_VER_IEEE) {880 iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_STREAM;881 ret = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app);882 if (!ret) {883 iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_ANY;884 ret = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app);885 }886 } else if (caps & DCB_CAP_DCBX_VER_CEE) {887 iscsi_dcb_app.selector = DCB_APP_IDTYPE_PORTNUM;888 889 ret = dcb_getapp(ndev, &iscsi_dcb_app);890 }891 892 pr_info("iSCSI priority is set to %u\n", cxgbit_select_priority(ret));893 894 return cxgbit_select_priority(ret);895}896#endif897 898static int899cxgbit_offload_init(struct cxgbit_sock *csk, int iptype, __u8 *peer_ip,900 u16 local_port, struct dst_entry *dst,901 struct cxgbit_device *cdev)902{903 struct neighbour *n;904 int ret, step;905 struct net_device *ndev;906 u16 rxq_idx, port_id;907#ifdef CONFIG_CHELSIO_T4_DCB908 u8 priority = 0;909#endif910 911 n = dst_neigh_lookup(dst, peer_ip);912 if (!n)913 return -ENODEV;914 915 rcu_read_lock();916 if (!(n->nud_state & NUD_VALID))917 neigh_event_send(n, NULL);918 919 ret = -ENOMEM;920 if (n->dev->flags & IFF_LOOPBACK) {921 if (iptype == 4)922 ndev = cxgbit_ipv4_netdev(*(__be32 *)peer_ip);923 else if (IS_ENABLED(CONFIG_IPV6))924 ndev = cxgbit_ipv6_netdev((struct in6_addr *)peer_ip);925 else926 ndev = NULL;927 928 if (!ndev) {929 ret = -ENODEV;930 goto out;931 }932 933 csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t,934 n, ndev, 0);935 if (!csk->l2t)936 goto out;937 csk->mtu = ndev->mtu;938 csk->tx_chan = cxgb4_port_chan(ndev);939 csk->smac_idx =940 ((struct port_info *)netdev_priv(ndev))->smt_idx;941 step = cdev->lldi.ntxq /942 cdev->lldi.nchan;943 csk->txq_idx = cxgb4_port_idx(ndev) * step;944 step = cdev->lldi.nrxq /945 cdev->lldi.nchan;946 csk->ctrlq_idx = cxgb4_port_idx(ndev);947 csk->rss_qid = cdev->lldi.rxq_ids[948 cxgb4_port_idx(ndev) * step];949 csk->port_id = cxgb4_port_idx(ndev);950 cxgbit_set_tcp_window(csk,951 (struct port_info *)netdev_priv(ndev));952 } else {953 ndev = cxgbit_get_real_dev(n->dev);954 if (!ndev) {955 ret = -ENODEV;956 goto out;957 }958 959#ifdef CONFIG_CHELSIO_T4_DCB960 if (cxgbit_get_iscsi_dcb_state(ndev))961 priority = cxgbit_get_iscsi_dcb_priority(ndev,962 local_port);963 964 csk->dcb_priority = priority;965 966 csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t, n, ndev, priority);967#else968 csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t, n, ndev, 0);969#endif970 if (!csk->l2t)971 goto out;972 port_id = cxgb4_port_idx(ndev);973 csk->mtu = dst_mtu(dst);974 csk->tx_chan = cxgb4_port_chan(ndev);975 csk->smac_idx =976 ((struct port_info *)netdev_priv(ndev))->smt_idx;977 step = cdev->lldi.ntxq /978 cdev->lldi.nports;979 csk->txq_idx = (port_id * step) +980 (cdev->selectq[port_id][0]++ % step);981 csk->ctrlq_idx = cxgb4_port_idx(ndev);982 step = cdev->lldi.nrxq /983 cdev->lldi.nports;984 rxq_idx = (port_id * step) +985 (cdev->selectq[port_id][1]++ % step);986 csk->rss_qid = cdev->lldi.rxq_ids[rxq_idx];987 csk->port_id = port_id;988 cxgbit_set_tcp_window(csk,989 (struct port_info *)netdev_priv(ndev));990 }991 ret = 0;992out:993 rcu_read_unlock();994 neigh_release(n);995 return ret;996}997 998int cxgbit_ofld_send(struct cxgbit_device *cdev, struct sk_buff *skb)999{1000 int ret = 0;1001 1002 if (!test_bit(CDEV_STATE_UP, &cdev->flags)) {1003 kfree_skb(skb);1004 pr_err("%s - device not up - dropping\n", __func__);1005 return -EIO;1006 }1007 1008 ret = cxgb4_ofld_send(cdev->lldi.ports[0], skb);1009 if (ret < 0)1010 kfree_skb(skb);1011 return ret < 0 ? ret : 0;1012}1013 1014static void cxgbit_release_tid(struct cxgbit_device *cdev, u32 tid)1015{1016 u32 len = roundup(sizeof(struct cpl_tid_release), 16);1017 struct sk_buff *skb;1018 1019 skb = alloc_skb(len, GFP_ATOMIC);1020 if (!skb)1021 return;1022 1023 cxgb_mk_tid_release(skb, len, tid, 0);1024 cxgbit_ofld_send(cdev, skb);1025}1026 1027int1028cxgbit_l2t_send(struct cxgbit_device *cdev, struct sk_buff *skb,1029 struct l2t_entry *l2e)1030{1031 int ret = 0;1032 1033 if (!test_bit(CDEV_STATE_UP, &cdev->flags)) {1034 kfree_skb(skb);1035 pr_err("%s - device not up - dropping\n", __func__);1036 return -EIO;1037 }1038 1039 ret = cxgb4_l2t_send(cdev->lldi.ports[0], skb, l2e);1040 if (ret < 0)1041 kfree_skb(skb);1042 return ret < 0 ? ret : 0;1043}1044 1045static void cxgbit_send_rx_credits(struct cxgbit_sock *csk, struct sk_buff *skb)1046{1047 if (csk->com.state != CSK_STATE_ESTABLISHED) {1048 __kfree_skb(skb);1049 return;1050 }1051 1052 cxgbit_ofld_send(csk->com.cdev, skb);1053}1054 1055/*1056 * CPL connection rx data ack: host ->1057 * Send RX credits through an RX_DATA_ACK CPL message.1058 * Returns the number of credits sent.1059 */1060int cxgbit_rx_data_ack(struct cxgbit_sock *csk)1061{1062 struct sk_buff *skb;1063 u32 len = roundup(sizeof(struct cpl_rx_data_ack), 16);1064 u32 credit_dack;1065 1066 skb = alloc_skb(len, GFP_KERNEL);1067 if (!skb)1068 return -1;1069 1070 credit_dack = RX_DACK_CHANGE_F | RX_DACK_MODE_V(3) |1071 RX_CREDITS_V(csk->rx_credits);1072 1073 cxgb_mk_rx_data_ack(skb, len, csk->tid, csk->ctrlq_idx,1074 credit_dack);1075 1076 csk->rx_credits = 0;1077 1078 spin_lock_bh(&csk->lock);1079 if (csk->lock_owner) {1080 cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_send_rx_credits;1081 __skb_queue_tail(&csk->backlogq, skb);1082 spin_unlock_bh(&csk->lock);1083 return 0;1084 }1085 1086 cxgbit_send_rx_credits(csk, skb);1087 spin_unlock_bh(&csk->lock);1088 1089 return 0;1090}1091 1092#define FLOWC_WR_NPARAMS_MIN 91093#define FLOWC_WR_NPARAMS_MAX 111094static int cxgbit_alloc_csk_skb(struct cxgbit_sock *csk)1095{1096 struct sk_buff *skb;1097 u32 len, flowclen;1098 u8 i;1099 1100 flowclen = offsetof(struct fw_flowc_wr,1101 mnemval[FLOWC_WR_NPARAMS_MAX]);1102 1103 len = max_t(u32, sizeof(struct cpl_abort_req),1104 sizeof(struct cpl_abort_rpl));1105 1106 len = max(len, flowclen);1107 len = roundup(len, 16);1108 1109 for (i = 0; i < 3; i++) {1110 skb = alloc_skb(len, GFP_ATOMIC);1111 if (!skb)1112 goto out;1113 __skb_queue_tail(&csk->skbq, skb);1114 }1115 1116 skb = alloc_skb(LRO_SKB_MIN_HEADROOM, GFP_ATOMIC);1117 if (!skb)1118 goto out;1119 1120 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);1121 csk->lro_hskb = skb;1122 1123 return 0;1124out:1125 __skb_queue_purge(&csk->skbq);1126 return -ENOMEM;1127}1128 1129static void1130cxgbit_pass_accept_rpl(struct cxgbit_sock *csk, struct cpl_pass_accept_req *req)1131{1132 struct sk_buff *skb;1133 const struct tcphdr *tcph;1134 struct cpl_t5_pass_accept_rpl *rpl5;1135 struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;1136 unsigned int len = roundup(sizeof(*rpl5), 16);1137 unsigned int mtu_idx;1138 u64 opt0;1139 u32 opt2, hlen;1140 u32 wscale;1141 u32 win;1142 1143 pr_debug("%s csk %p tid %u\n", __func__, csk, csk->tid);1144 1145 skb = alloc_skb(len, GFP_ATOMIC);1146 if (!skb) {1147 cxgbit_put_csk(csk);1148 return;1149 }1150 1151 rpl5 = __skb_put_zero(skb, len);1152 1153 INIT_TP_WR(rpl5, csk->tid);1154 OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,1155 csk->tid));1156 cxgb_best_mtu(csk->com.cdev->lldi.mtus, csk->mtu, &mtu_idx,1157 req->tcpopt.tstamp,1158 (csk->com.remote_addr.ss_family == AF_INET) ? 0 : 1);1159 wscale = cxgb_compute_wscale(csk->rcv_win);1160 /*1161 * Specify the largest window that will fit in opt0. The1162 * remainder will be specified in the rx_data_ack.1163 */1164 win = csk->rcv_win >> 10;1165 if (win > RCV_BUFSIZ_M)1166 win = RCV_BUFSIZ_M;1167 opt0 = TCAM_BYPASS_F |1168 WND_SCALE_V(wscale) |1169 MSS_IDX_V(mtu_idx) |1170 L2T_IDX_V(csk->l2t->idx) |1171 TX_CHAN_V(csk->tx_chan) |1172 SMAC_SEL_V(csk->smac_idx) |1173 DSCP_V(csk->tos >> 2) |1174 ULP_MODE_V(ULP_MODE_ISCSI) |1175 RCV_BUFSIZ_V(win);1176 1177 opt2 = RX_CHANNEL_V(0) |1178 RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);1179 1180 if (!is_t5(lldi->adapter_type))1181 opt2 |= RX_FC_DISABLE_F;1182 1183 if (req->tcpopt.tstamp)1184 opt2 |= TSTAMPS_EN_F;1185 if (req->tcpopt.sack)1186 opt2 |= SACK_EN_F;1187 if (wscale)1188 opt2 |= WND_SCALE_EN_F;1189 1190 hlen = ntohl(req->hdr_len);1191 1192 if (is_t5(lldi->adapter_type))1193 tcph = (struct tcphdr *)((u8 *)(req + 1) +1194 ETH_HDR_LEN_G(hlen) + IP_HDR_LEN_G(hlen));1195 else1196 tcph = (struct tcphdr *)((u8 *)(req + 1) +1197 T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));1198 1199 if (tcph->ece && tcph->cwr)1200 opt2 |= CCTRL_ECN_V(1);1201 1202 opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);1203 1204 opt2 |= T5_ISS_F;1205 rpl5->iss = cpu_to_be32((get_random_u32() & ~7UL) - 1);1206 1207 opt2 |= T5_OPT_2_VALID_F;1208 1209 rpl5->opt0 = cpu_to_be64(opt0);1210 rpl5->opt2 = cpu_to_be32(opt2);1211 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->ctrlq_idx);1212 t4_set_arp_err_handler(skb, csk, cxgbit_arp_failure_discard);1213 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);1214}1215 1216static void1217cxgbit_pass_accept_req(struct cxgbit_device *cdev, struct sk_buff *skb)1218{1219 struct cxgbit_sock *csk = NULL;1220 struct cxgbit_np *cnp;1221 struct cpl_pass_accept_req *req = cplhdr(skb);1222 unsigned int stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));1223 struct tid_info *t = cdev->lldi.tids;1224 unsigned int tid = GET_TID(req);1225 u16 peer_mss = ntohs(req->tcpopt.mss);1226 unsigned short hdrs;1227 1228 struct dst_entry *dst;1229 __u8 local_ip[16], peer_ip[16];1230 __be16 local_port, peer_port;1231 int ret;1232 int iptype;1233 1234 pr_debug("%s: cdev = %p; stid = %u; tid = %u\n",1235 __func__, cdev, stid, tid);1236 1237 cnp = lookup_stid(t, stid);1238 if (!cnp) {1239 pr_err("%s connect request on invalid stid %d\n",1240 __func__, stid);1241 goto rel_skb;1242 }1243 1244 if (cnp->com.state != CSK_STATE_LISTEN) {1245 pr_err("%s - listening parent not in CSK_STATE_LISTEN\n",1246 __func__);1247 goto reject;1248 }1249 1250 csk = lookup_tid(t, tid);1251 if (csk) {1252 pr_err("%s csk not null tid %u\n",1253 __func__, tid);1254 goto rel_skb;1255 }1256 1257 cxgb_get_4tuple(req, cdev->lldi.adapter_type, &iptype, local_ip,1258 peer_ip, &local_port, &peer_port);1259 1260 /* Find output route */1261 if (iptype == 4) {1262 pr_debug("%s parent sock %p tid %u laddr %pI4 raddr %pI4 "1263 "lport %d rport %d peer_mss %d\n"1264 , __func__, cnp, tid,1265 local_ip, peer_ip, ntohs(local_port),1266 ntohs(peer_port), peer_mss);1267 dst = cxgb_find_route(&cdev->lldi, cxgbit_get_real_dev,1268 *(__be32 *)local_ip,1269 *(__be32 *)peer_ip,1270 local_port, peer_port,1271 PASS_OPEN_TOS_G(ntohl(req->tos_stid)));1272 } else {1273 pr_debug("%s parent sock %p tid %u laddr %pI6 raddr %pI6 "1274 "lport %d rport %d peer_mss %d\n"1275 , __func__, cnp, tid,1276 local_ip, peer_ip, ntohs(local_port),1277 ntohs(peer_port), peer_mss);1278 dst = cxgb_find_route6(&cdev->lldi, cxgbit_get_real_dev,1279 local_ip, peer_ip,1280 local_port, peer_port,1281 PASS_OPEN_TOS_G(ntohl(req->tos_stid)),1282 ((struct sockaddr_in6 *)1283 &cnp->com.local_addr)->sin6_scope_id);1284 }1285 if (!dst) {1286 pr_err("%s - failed to find dst entry!\n",1287 __func__);1288 goto reject;1289 }1290 1291 csk = kzalloc(sizeof(*csk), GFP_ATOMIC);1292 if (!csk) {1293 dst_release(dst);1294 goto rel_skb;1295 }1296 1297 ret = cxgbit_offload_init(csk, iptype, peer_ip, ntohs(local_port),1298 dst, cdev);1299 if (ret) {1300 pr_err("%s - failed to allocate l2t entry!\n",1301 __func__);1302 dst_release(dst);1303 kfree(csk);1304 goto reject;1305 }1306 1307 kref_init(&csk->kref);1308 init_completion(&csk->com.wr_wait.completion);1309 1310 INIT_LIST_HEAD(&csk->accept_node);1311 1312 hdrs = (iptype == 4 ? sizeof(struct iphdr) : sizeof(struct ipv6hdr)) +1313 sizeof(struct tcphdr) + (req->tcpopt.tstamp ? 12 : 0);1314 if (peer_mss && csk->mtu > (peer_mss + hdrs))1315 csk->mtu = peer_mss + hdrs;1316 1317 csk->com.state = CSK_STATE_CONNECTING;1318 csk->com.cdev = cdev;1319 csk->cnp = cnp;1320 csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid));1321 csk->dst = dst;1322 csk->tid = tid;1323 csk->wr_cred = cdev->lldi.wr_cred -1324 DIV_ROUND_UP(sizeof(struct cpl_abort_req), 16);1325 csk->wr_max_cred = csk->wr_cred;1326 csk->wr_una_cred = 0;1327 1328 if (iptype == 4) {1329 struct sockaddr_in *sin = (struct sockaddr_in *)1330 &csk->com.local_addr;1331 sin->sin_family = AF_INET;1332 sin->sin_port = local_port;1333 sin->sin_addr.s_addr = *(__be32 *)local_ip;1334 1335 sin = (struct sockaddr_in *)&csk->com.remote_addr;1336 sin->sin_family = AF_INET;1337 sin->sin_port = peer_port;1338 sin->sin_addr.s_addr = *(__be32 *)peer_ip;1339 } else {1340 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)1341 &csk->com.local_addr;1342 1343 sin6->sin6_family = PF_INET6;1344 sin6->sin6_port = local_port;1345 memcpy(sin6->sin6_addr.s6_addr, local_ip, 16);1346 cxgb4_clip_get(cdev->lldi.ports[0],1347 (const u32 *)&sin6->sin6_addr.s6_addr,1348 1);1349 1350 sin6 = (struct sockaddr_in6 *)&csk->com.remote_addr;1351 sin6->sin6_family = PF_INET6;1352 sin6->sin6_port = peer_port;1353 memcpy(sin6->sin6_addr.s6_addr, peer_ip, 16);1354 }1355 1356 skb_queue_head_init(&csk->rxq);1357 skb_queue_head_init(&csk->txq);1358 skb_queue_head_init(&csk->ppodq);1359 skb_queue_head_init(&csk->backlogq);1360 skb_queue_head_init(&csk->skbq);1361 cxgbit_sock_reset_wr_list(csk);1362 spin_lock_init(&csk->lock);1363 init_waitqueue_head(&csk->waitq);1364 csk->lock_owner = false;1365 1366 if (cxgbit_alloc_csk_skb(csk)) {1367 dst_release(dst);1368 kfree(csk);1369 goto rel_skb;1370 }1371 1372 cxgbit_get_cnp(cnp);1373 cxgbit_get_cdev(cdev);1374 1375 spin_lock(&cdev->cskq.lock);1376 list_add_tail(&csk->list, &cdev->cskq.list);1377 spin_unlock(&cdev->cskq.lock);1378 cxgb4_insert_tid(t, csk, tid, csk->com.local_addr.ss_family);1379 cxgbit_pass_accept_rpl(csk, req);1380 goto rel_skb;1381 1382reject:1383 cxgbit_release_tid(cdev, tid);1384rel_skb:1385 __kfree_skb(skb);1386}1387 1388static u321389cxgbit_tx_flowc_wr_credits(struct cxgbit_sock *csk, u32 *nparamsp,1390 u32 *flowclenp)1391{1392 u32 nparams, flowclen16, flowclen;1393 1394 nparams = FLOWC_WR_NPARAMS_MIN;1395 1396 if (csk->snd_wscale)1397 nparams++;1398 1399#ifdef CONFIG_CHELSIO_T4_DCB1400 nparams++;1401#endif1402 flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);1403 flowclen16 = DIV_ROUND_UP(flowclen, 16);1404 flowclen = flowclen16 * 16;1405 /*1406 * Return the number of 16-byte credits used by the flowc request.1407 * Pass back the nparams and actual flowc length if requested.1408 */1409 if (nparamsp)1410 *nparamsp = nparams;1411 if (flowclenp)1412 *flowclenp = flowclen;1413 return flowclen16;1414}1415 1416u32 cxgbit_send_tx_flowc_wr(struct cxgbit_sock *csk)1417{1418 struct cxgbit_device *cdev = csk->com.cdev;1419 struct fw_flowc_wr *flowc;1420 u32 nparams, flowclen16, flowclen;1421 struct sk_buff *skb;1422 u8 index;1423 1424#ifdef CONFIG_CHELSIO_T4_DCB1425 u16 vlan = ((struct l2t_entry *)csk->l2t)->vlan;1426#endif1427 1428 flowclen16 = cxgbit_tx_flowc_wr_credits(csk, &nparams, &flowclen);1429 1430 skb = __skb_dequeue(&csk->skbq);1431 flowc = __skb_put_zero(skb, flowclen);1432 1433 flowc->op_to_nparams = cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR) |1434 FW_FLOWC_WR_NPARAMS_V(nparams));1435 flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16_V(flowclen16) |1436 FW_WR_FLOWID_V(csk->tid));1437 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;1438 flowc->mnemval[0].val = cpu_to_be32(FW_PFVF_CMD_PFN_V1439 (csk->com.cdev->lldi.pf));1440 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;1441 flowc->mnemval[1].val = cpu_to_be32(csk->tx_chan);1442 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;1443 flowc->mnemval[2].val = cpu_to_be32(csk->tx_chan);1444 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;1445 flowc->mnemval[3].val = cpu_to_be32(csk->rss_qid);1446 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;1447 flowc->mnemval[4].val = cpu_to_be32(csk->snd_nxt);1448 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;1449 flowc->mnemval[5].val = cpu_to_be32(csk->rcv_nxt);1450 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;1451 flowc->mnemval[6].val = cpu_to_be32(csk->snd_win);1452 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;1453 flowc->mnemval[7].val = cpu_to_be32(csk->emss);1454 1455 flowc->mnemval[8].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX;1456 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags))1457 flowc->mnemval[8].val = cpu_to_be32(CXGBIT_MAX_ISO_PAYLOAD);1458 else1459 flowc->mnemval[8].val = cpu_to_be32(16384);1460 1461 index = 9;1462 1463 if (csk->snd_wscale) {1464 flowc->mnemval[index].mnemonic = FW_FLOWC_MNEM_RCV_SCALE;1465 flowc->mnemval[index].val = cpu_to_be32(csk->snd_wscale);1466 index++;1467 }1468 1469#ifdef CONFIG_CHELSIO_T4_DCB1470 flowc->mnemval[index].mnemonic = FW_FLOWC_MNEM_DCBPRIO;1471 if (vlan == VLAN_NONE) {1472 pr_warn("csk %u without VLAN Tag on DCB Link\n", csk->tid);1473 flowc->mnemval[index].val = cpu_to_be32(0);1474 } else1475 flowc->mnemval[index].val = cpu_to_be32(1476 (vlan & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT);1477#endif1478 1479 pr_debug("%s: csk %p; tx_chan = %u; rss_qid = %u; snd_seq = %u;"1480 " rcv_seq = %u; snd_win = %u; emss = %u\n",1481 __func__, csk, csk->tx_chan, csk->rss_qid, csk->snd_nxt,1482 csk->rcv_nxt, csk->snd_win, csk->emss);1483 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);1484 cxgbit_ofld_send(csk->com.cdev, skb);1485 return flowclen16;1486}1487 1488static int1489cxgbit_send_tcb_skb(struct cxgbit_sock *csk, struct sk_buff *skb)1490{1491 spin_lock_bh(&csk->lock);1492 if (unlikely(csk->com.state != CSK_STATE_ESTABLISHED)) {1493 spin_unlock_bh(&csk->lock);1494 pr_err("%s: csk 0x%p, tid %u, state %u\n",1495 __func__, csk, csk->tid, csk->com.state);1496 __kfree_skb(skb);1497 return -1;1498 }1499 1500 cxgbit_get_csk(csk);1501 cxgbit_init_wr_wait(&csk->com.wr_wait);1502 cxgbit_ofld_send(csk->com.cdev, skb);1503 spin_unlock_bh(&csk->lock);1504 1505 return 0;1506}1507 1508int cxgbit_setup_conn_digest(struct cxgbit_sock *csk)1509{1510 struct sk_buff *skb;1511 struct cpl_set_tcb_field *req;1512 u8 hcrc = csk->submode & CXGBIT_SUBMODE_HCRC;1513 u8 dcrc = csk->submode & CXGBIT_SUBMODE_DCRC;1514 unsigned int len = roundup(sizeof(*req), 16);1515 int ret;1516 1517 skb = alloc_skb(len, GFP_KERNEL);1518 if (!skb)1519 return -ENOMEM;1520 1521 /* set up ulp submode */1522 req = __skb_put_zero(skb, len);1523 1524 INIT_TP_WR(req, csk->tid);1525 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid));1526 req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid));1527 req->word_cookie = htons(0);1528 req->mask = cpu_to_be64(0x3 << 4);1529 req->val = cpu_to_be64(((hcrc ? ULP_CRC_HEADER : 0) |1530 (dcrc ? ULP_CRC_DATA : 0)) << 4);1531 set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->ctrlq_idx);1532 1533 if (cxgbit_send_tcb_skb(csk, skb))1534 return -1;1535 1536 ret = cxgbit_wait_for_reply(csk->com.cdev,1537 &csk->com.wr_wait,1538 csk->tid, 5, __func__);1539 if (ret)1540 return -1;1541 1542 return 0;1543}1544 1545int cxgbit_setup_conn_pgidx(struct cxgbit_sock *csk, u32 pg_idx)1546{1547 struct sk_buff *skb;1548 struct cpl_set_tcb_field *req;1549 unsigned int len = roundup(sizeof(*req), 16);1550 int ret;1551 1552 skb = alloc_skb(len, GFP_KERNEL);1553 if (!skb)1554 return -ENOMEM;1555 1556 req = __skb_put_zero(skb, len);1557 1558 INIT_TP_WR(req, csk->tid);1559 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid));1560 req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid));1561 req->word_cookie = htons(0);1562 req->mask = cpu_to_be64(0x3 << 8);1563 req->val = cpu_to_be64(pg_idx << 8);1564 set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->ctrlq_idx);1565 1566 if (cxgbit_send_tcb_skb(csk, skb))1567 return -1;1568 1569 ret = cxgbit_wait_for_reply(csk->com.cdev,1570 &csk->com.wr_wait,1571 csk->tid, 5, __func__);1572 if (ret)1573 return -1;1574 1575 return 0;1576}1577 1578static void1579cxgbit_pass_open_rpl(struct cxgbit_device *cdev, struct sk_buff *skb)1580{1581 struct cpl_pass_open_rpl *rpl = cplhdr(skb);1582 struct tid_info *t = cdev->lldi.tids;1583 unsigned int stid = GET_TID(rpl);1584 struct cxgbit_np *cnp = lookup_stid(t, stid);1585 1586 pr_debug("%s: cnp = %p; stid = %u; status = %d\n",1587 __func__, cnp, stid, rpl->status);1588 1589 if (!cnp) {1590 pr_info("%s stid %d lookup failure\n", __func__, stid);1591 goto rel_skb;1592 }1593 1594 cxgbit_wake_up(&cnp->com.wr_wait, __func__, rpl->status);1595 cxgbit_put_cnp(cnp);1596rel_skb:1597 __kfree_skb(skb);1598}1599 1600static void1601cxgbit_close_listsrv_rpl(struct cxgbit_device *cdev, struct sk_buff *skb)1602{1603 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb);1604 struct tid_info *t = cdev->lldi.tids;1605 unsigned int stid = GET_TID(rpl);1606 struct cxgbit_np *cnp = lookup_stid(t, stid);1607 1608 pr_debug("%s: cnp = %p; stid = %u; status = %d\n",1609 __func__, cnp, stid, rpl->status);1610 1611 if (!cnp) {1612 pr_info("%s stid %d lookup failure\n", __func__, stid);1613 goto rel_skb;1614 }1615 1616 cxgbit_wake_up(&cnp->com.wr_wait, __func__, rpl->status);1617 cxgbit_put_cnp(cnp);1618rel_skb:1619 __kfree_skb(skb);1620}1621 1622static void1623cxgbit_pass_establish(struct cxgbit_device *cdev, struct sk_buff *skb)1624{1625 struct cpl_pass_establish *req = cplhdr(skb);1626 struct tid_info *t = cdev->lldi.tids;1627 unsigned int tid = GET_TID(req);1628 struct cxgbit_sock *csk;1629 struct cxgbit_np *cnp;1630 u16 tcp_opt = be16_to_cpu(req->tcp_opt);1631 u32 snd_isn = be32_to_cpu(req->snd_isn);1632 u32 rcv_isn = be32_to_cpu(req->rcv_isn);1633 1634 csk = lookup_tid(t, tid);1635 if (unlikely(!csk)) {1636 pr_err("can't find connection for tid %u.\n", tid);1637 goto rel_skb;1638 }1639 cnp = csk->cnp;1640 1641 pr_debug("%s: csk %p; tid %u; cnp %p\n",1642 __func__, csk, tid, cnp);1643 1644 csk->write_seq = snd_isn;1645 csk->snd_una = snd_isn;1646 csk->snd_nxt = snd_isn;1647 1648 csk->rcv_nxt = rcv_isn;1649 1650 csk->snd_wscale = TCPOPT_SND_WSCALE_G(tcp_opt);1651 cxgbit_set_emss(csk, tcp_opt);1652 dst_confirm(csk->dst);1653 csk->com.state = CSK_STATE_ESTABLISHED;1654 spin_lock_bh(&cnp->np_accept_lock);1655 list_add_tail(&csk->accept_node, &cnp->np_accept_list);1656 spin_unlock_bh(&cnp->np_accept_lock);1657 complete(&cnp->accept_comp);1658rel_skb:1659 __kfree_skb(skb);1660}1661 1662static void cxgbit_queue_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)1663{1664 cxgbit_skcb_flags(skb) = 0;1665 spin_lock_bh(&csk->rxq.lock);1666 __skb_queue_tail(&csk->rxq, skb);1667 spin_unlock_bh(&csk->rxq.lock);1668 wake_up(&csk->waitq);1669}1670 1671static void cxgbit_peer_close(struct cxgbit_sock *csk, struct sk_buff *skb)1672{1673 pr_debug("%s: csk %p; tid %u; state %d\n",1674 __func__, csk, csk->tid, csk->com.state);1675 1676 switch (csk->com.state) {1677 case CSK_STATE_ESTABLISHED:1678 csk->com.state = CSK_STATE_CLOSING;1679 cxgbit_queue_rx_skb(csk, skb);1680 return;1681 case CSK_STATE_CLOSING:1682 /* simultaneous close */1683 csk->com.state = CSK_STATE_MORIBUND;1684 break;1685 case CSK_STATE_MORIBUND:1686 csk->com.state = CSK_STATE_DEAD;1687 cxgbit_put_csk(csk);1688 break;1689 case CSK_STATE_ABORTING:1690 break;1691 default:1692 pr_info("%s: cpl_peer_close in bad state %d\n",1693 __func__, csk->com.state);1694 }1695 1696 __kfree_skb(skb);1697}1698 1699static void cxgbit_close_con_rpl(struct cxgbit_sock *csk, struct sk_buff *skb)1700{1701 pr_debug("%s: csk %p; tid %u; state %d\n",1702 __func__, csk, csk->tid, csk->com.state);1703 1704 switch (csk->com.state) {1705 case CSK_STATE_CLOSING:1706 csk->com.state = CSK_STATE_MORIBUND;1707 break;1708 case CSK_STATE_MORIBUND:1709 csk->com.state = CSK_STATE_DEAD;1710 cxgbit_put_csk(csk);1711 break;1712 case CSK_STATE_ABORTING:1713 case CSK_STATE_DEAD:1714 break;1715 default:1716 pr_info("%s: cpl_close_con_rpl in bad state %d\n",1717 __func__, csk->com.state);1718 }1719 1720 __kfree_skb(skb);1721}1722 1723static void cxgbit_abort_req_rss(struct cxgbit_sock *csk, struct sk_buff *skb)1724{1725 struct cpl_abort_req_rss *hdr = cplhdr(skb);1726 unsigned int tid = GET_TID(hdr);1727 struct sk_buff *rpl_skb;1728 bool release = false;1729 bool wakeup_thread = false;1730 u32 len = roundup(sizeof(struct cpl_abort_rpl), 16);1731 1732 pr_debug("%s: csk %p; tid %u; state %d\n",1733 __func__, csk, tid, csk->com.state);1734 1735 if (cxgb_is_neg_adv(hdr->status)) {1736 pr_err("%s: got neg advise %d on tid %u\n",1737 __func__, hdr->status, tid);1738 goto rel_skb;1739 }1740 1741 switch (csk->com.state) {1742 case CSK_STATE_CONNECTING:1743 case CSK_STATE_MORIBUND:1744 csk->com.state = CSK_STATE_DEAD;1745 release = true;1746 break;1747 case CSK_STATE_ESTABLISHED:1748 csk->com.state = CSK_STATE_DEAD;1749 wakeup_thread = true;1750 break;1751 case CSK_STATE_CLOSING:1752 csk->com.state = CSK_STATE_DEAD;1753 if (!csk->conn)1754 release = true;1755 break;1756 case CSK_STATE_ABORTING:1757 break;1758 default:1759 pr_info("%s: cpl_abort_req_rss in bad state %d\n",1760 __func__, csk->com.state);1761 csk->com.state = CSK_STATE_DEAD;1762 }1763 1764 __skb_queue_purge(&csk->txq);1765 1766 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags))1767 cxgbit_send_tx_flowc_wr(csk);1768 1769 rpl_skb = __skb_dequeue(&csk->skbq);1770 1771 cxgb_mk_abort_rpl(rpl_skb, len, csk->tid, csk->txq_idx);1772 cxgbit_ofld_send(csk->com.cdev, rpl_skb);1773 1774 if (wakeup_thread) {1775 cxgbit_queue_rx_skb(csk, skb);1776 return;1777 }1778 1779 if (release)1780 cxgbit_put_csk(csk);1781rel_skb:1782 __kfree_skb(skb);1783}1784 1785static void cxgbit_abort_rpl_rss(struct cxgbit_sock *csk, struct sk_buff *skb)1786{1787 struct cpl_abort_rpl_rss *rpl = cplhdr(skb);1788 1789 pr_debug("%s: csk %p; tid %u; state %d\n",1790 __func__, csk, csk->tid, csk->com.state);1791 1792 switch (csk->com.state) {1793 case CSK_STATE_ABORTING:1794 csk->com.state = CSK_STATE_DEAD;1795 if (test_bit(CSK_ABORT_RPL_WAIT, &csk->com.flags))1796 cxgbit_wake_up(&csk->com.wr_wait, __func__,1797 rpl->status);1798 cxgbit_put_csk(csk);1799 break;1800 default:1801 pr_info("%s: cpl_abort_rpl_rss in state %d\n",1802 __func__, csk->com.state);1803 }1804 1805 __kfree_skb(skb);1806}1807 1808static bool cxgbit_credit_err(const struct cxgbit_sock *csk)1809{1810 const struct sk_buff *skb = csk->wr_pending_head;1811 u32 credit = 0;1812 1813 if (unlikely(csk->wr_cred > csk->wr_max_cred)) {1814 pr_err("csk 0x%p, tid %u, credit %u > %u\n",1815 csk, csk->tid, csk->wr_cred, csk->wr_max_cred);1816 return true;1817 }1818 1819 while (skb) {1820 credit += (__force u32)skb->csum;1821 skb = cxgbit_skcb_tx_wr_next(skb);1822 }1823 1824 if (unlikely((csk->wr_cred + credit) != csk->wr_max_cred)) {1825 pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n",1826 csk, csk->tid, csk->wr_cred,1827 credit, csk->wr_max_cred);1828 1829 return true;1830 }1831 1832 return false;1833}1834 1835static void cxgbit_fw4_ack(struct cxgbit_sock *csk, struct sk_buff *skb)1836{1837 struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)cplhdr(skb);1838 u32 credits = rpl->credits;1839 u32 snd_una = ntohl(rpl->snd_una);1840 1841 csk->wr_cred += credits;1842 if (csk->wr_una_cred > (csk->wr_max_cred - csk->wr_cred))1843 csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred;1844 1845 while (credits) {1846 struct sk_buff *p = cxgbit_sock_peek_wr(csk);1847 u32 csum;1848 1849 if (unlikely(!p)) {1850 pr_err("csk 0x%p,%u, cr %u,%u+%u, empty.\n",1851 csk, csk->tid, credits,1852 csk->wr_cred, csk->wr_una_cred);1853 break;1854 }1855 1856 csum = (__force u32)p->csum;1857 if (unlikely(credits < csum)) {1858 pr_warn("csk 0x%p,%u, cr %u,%u+%u, < %u.\n",1859 csk, csk->tid,1860 credits, csk->wr_cred, csk->wr_una_cred,1861 csum);1862 p->csum = (__force __wsum)(csum - credits);1863 break;1864 }1865 1866 cxgbit_sock_dequeue_wr(csk);1867 credits -= csum;1868 kfree_skb(p);1869 }1870 1871 if (unlikely(cxgbit_credit_err(csk))) {1872 cxgbit_queue_rx_skb(csk, skb);1873 return;1874 }1875 1876 if (rpl->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) {1877 if (unlikely(before(snd_una, csk->snd_una))) {1878 pr_warn("csk 0x%p,%u, snd_una %u/%u.",1879 csk, csk->tid, snd_una,1880 csk->snd_una);1881 goto rel_skb;1882 }1883 1884 if (csk->snd_una != snd_una) {1885 csk->snd_una = snd_una;1886 dst_confirm(csk->dst);1887 }1888 }1889 1890 if (skb_queue_len(&csk->txq))1891 cxgbit_push_tx_frames(csk);1892 1893rel_skb:1894 __kfree_skb(skb);1895}1896 1897static void cxgbit_set_tcb_rpl(struct cxgbit_device *cdev, struct sk_buff *skb)1898{1899 struct cxgbit_sock *csk;1900 struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data;1901 unsigned int tid = GET_TID(rpl);1902 struct cxgb4_lld_info *lldi = &cdev->lldi;1903 struct tid_info *t = lldi->tids;1904 1905 csk = lookup_tid(t, tid);1906 if (unlikely(!csk)) {1907 pr_err("can't find connection for tid %u.\n", tid);1908 goto rel_skb;1909 } else {1910 cxgbit_wake_up(&csk->com.wr_wait, __func__, rpl->status);1911 }1912 1913 cxgbit_put_csk(csk);1914rel_skb:1915 __kfree_skb(skb);1916}1917 1918static void cxgbit_rx_data(struct cxgbit_device *cdev, struct sk_buff *skb)1919{1920 struct cxgbit_sock *csk;1921 struct cpl_rx_data *cpl = cplhdr(skb);1922 unsigned int tid = GET_TID(cpl);1923 struct cxgb4_lld_info *lldi = &cdev->lldi;1924 struct tid_info *t = lldi->tids;1925 1926 csk = lookup_tid(t, tid);1927 if (unlikely(!csk)) {1928 pr_err("can't find conn. for tid %u.\n", tid);1929 goto rel_skb;1930 }1931 1932 cxgbit_queue_rx_skb(csk, skb);1933 return;1934rel_skb:1935 __kfree_skb(skb);1936}1937 1938static void1939__cxgbit_process_rx_cpl(struct cxgbit_sock *csk, struct sk_buff *skb)1940{1941 spin_lock(&csk->lock);1942 if (csk->lock_owner) {1943 __skb_queue_tail(&csk->backlogq, skb);1944 spin_unlock(&csk->lock);1945 return;1946 }1947 1948 cxgbit_skcb_rx_backlog_fn(skb)(csk, skb);1949 spin_unlock(&csk->lock);1950}1951 1952static void cxgbit_process_rx_cpl(struct cxgbit_sock *csk, struct sk_buff *skb)1953{1954 cxgbit_get_csk(csk);1955 __cxgbit_process_rx_cpl(csk, skb);1956 cxgbit_put_csk(csk);1957}1958 1959static void cxgbit_rx_cpl(struct cxgbit_device *cdev, struct sk_buff *skb)1960{1961 struct cxgbit_sock *csk;1962 struct cpl_tx_data *cpl = cplhdr(skb);1963 struct cxgb4_lld_info *lldi = &cdev->lldi;1964 struct tid_info *t = lldi->tids;1965 unsigned int tid = GET_TID(cpl);1966 u8 opcode = cxgbit_skcb_rx_opcode(skb);1967 bool ref = true;1968 1969 switch (opcode) {1970 case CPL_FW4_ACK:1971 cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_fw4_ack;1972 ref = false;1973 break;1974 case CPL_PEER_CLOSE:1975 cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_peer_close;1976 break;1977 case CPL_CLOSE_CON_RPL:1978 cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_close_con_rpl;1979 break;1980 case CPL_ABORT_REQ_RSS:1981 cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_abort_req_rss;1982 break;1983 case CPL_ABORT_RPL_RSS:1984 cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_abort_rpl_rss;1985 break;1986 default:1987 goto rel_skb;1988 }1989 1990 csk = lookup_tid(t, tid);1991 if (unlikely(!csk)) {1992 pr_err("can't find conn. for tid %u.\n", tid);1993 goto rel_skb;1994 }1995 1996 if (ref)1997 cxgbit_process_rx_cpl(csk, skb);1998 else1999 __cxgbit_process_rx_cpl(csk, skb);2000 2001 return;2002rel_skb:2003 __kfree_skb(skb);2004}2005 2006cxgbit_cplhandler_func cxgbit_cplhandlers[NUM_CPL_CMDS] = {2007 [CPL_PASS_OPEN_RPL] = cxgbit_pass_open_rpl,2008 [CPL_CLOSE_LISTSRV_RPL] = cxgbit_close_listsrv_rpl,2009 [CPL_PASS_ACCEPT_REQ] = cxgbit_pass_accept_req,2010 [CPL_PASS_ESTABLISH] = cxgbit_pass_establish,2011 [CPL_SET_TCB_RPL] = cxgbit_set_tcb_rpl,2012 [CPL_RX_DATA] = cxgbit_rx_data,2013 [CPL_FW4_ACK] = cxgbit_rx_cpl,2014 [CPL_PEER_CLOSE] = cxgbit_rx_cpl,2015 [CPL_CLOSE_CON_RPL] = cxgbit_rx_cpl,2016 [CPL_ABORT_REQ_RSS] = cxgbit_rx_cpl,2017 [CPL_ABORT_RPL_RSS] = cxgbit_rx_cpl,2018};2019